diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py index 2c4fe8d35ff2..3ccd3e8415e7 100644 --- a/agent/monitoring/cron_health.py +++ b/agent/monitoring/cron_health.py @@ -25,7 +25,12 @@ _KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} _KNOWN_SOURCES = {"builtin", "direct", "external"} _KNOWN_DELIVERY_OUTCOMES = { - "delivered", "failed", "suppressed", "suppressed_acked", "not_configured", + "delivered", + "failed", + "unknown", + "suppressed", + "suppressed_acked", + "not_configured", } diff --git a/cron/executions.py b/cron/executions.py index 023ed75a4d42..d78ea194d8d8 100644 --- a/cron/executions.py +++ b/cron/executions.py @@ -7,12 +7,17 @@ from __future__ import annotations +import hashlib +import json import os import sqlite3 import threading +import unicodedata import time import uuid from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Any, Dict, Iterator, List, Optional from hermes_constants import get_hermes_home @@ -23,12 +28,33 @@ # profile's execution records into the import-time home. EXECUTIONS_FILE: Optional[Path] = None MAX_TERMINAL_EXECUTIONS = 1000 +_RECEIPT_SCHEMA_VERSION = 5 HANDOFF_ADOPTION_GRACE_SECONDS = 30.0 _TERMINAL_STATES = ("completed", "failed", "unknown") +_EXECUTION_ERROR_KINDS = frozenset({ + "blocked_config", + "claim_lost", + "dispatch_failed", + "execution_failed", + "interrupted", + "legacy_redacted", + "unknown", +}) _lock = threading.RLock() _PROCESS_ID = uuid.uuid4().hex +def scheduled_fire_identity(job_id: str, scheduled_for: str) -> str: + """Return a stable opaque identity for one job's scheduled occurrence.""" + normalized_job_id = _bounded_text(job_id, field="job_id", limit=256) + normalized_time = _bounded_text(scheduled_for, field="scheduled_for", limit=128) + parsed = datetime.fromisoformat(normalized_time.replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.utcoffset() is None: + parsed = parsed.replace(tzinfo=timezone.utc) + canonical = parsed.astimezone(timezone.utc).isoformat() + return hashlib.sha256(f"{normalized_job_id}\0{canonical}".encode("utf-8")).hexdigest() + + def _connect() -> sqlite3.Connection: from cron.jobs import _ensure_cron_dir @@ -42,8 +68,11 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: conn.row_factory = sqlite3.Row conn.execute("PRAGMA busy_timeout=5000") + conn.execute("PRAGMA foreign_keys=ON") apply_wal_with_fallback(conn, db_label="cron/executions.db") conn.execute("PRAGMA synchronous=FULL") + # Keep every schema migration and privacy scrub failure-atomic. + conn.execute("BEGIN IMMEDIATE") conn.execute( """CREATE TABLE IF NOT EXISTS executions ( id TEXT PRIMARY KEY, @@ -59,11 +88,17 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: claimed_at TEXT NOT NULL, started_at TEXT, finished_at TEXT, - error TEXT + error TEXT, + error_kind TEXT, + receipt_state TEXT )""" ) from hermes_cli.sqlite_util import add_column_if_missing + add_column_if_missing(conn, "executions", "fire_identity", "fire_identity TEXT") + add_column_if_missing(conn, "executions", "error_kind", "error_kind TEXT") + # NULL identifies executions written before the receipt contract. + add_column_if_missing(conn, "executions", "receipt_state", "receipt_state TEXT") add_column_if_missing( conn, "executions", "handoff_pending", "handoff_pending INTEGER NOT NULL DEFAULT 0", @@ -71,6 +106,12 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: add_column_if_missing( conn, "executions", "handoff_started_at", "handoff_started_at REAL" ) + # Old execution rows may contain provider text, message fragments, paths, + # or other sensitive diagnostics. Preserve only a bounded category. + conn.execute( + """UPDATE executions SET error_kind='legacy_redacted', error=NULL + WHERE error IS NOT NULL""" + ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_executions_job_claimed " "ON executions(job_id, claimed_at DESC, id DESC)" @@ -79,6 +120,72 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: "CREATE INDEX IF NOT EXISTS idx_executions_status_claimed " "ON executions(status, claimed_at DESC, id DESC)" ) + # A singleton metadata row avoids the old MAX(version) ambiguity. Migrate + # the prior multi-row table transactionally; its only data is the version. + receipt_columns = { + row["name"] for row in conn.execute("PRAGMA table_info(receipt_schema)").fetchall() + } + if receipt_columns and "singleton" not in receipt_columns: + conn.execute("DROP TABLE receipt_schema") + conn.execute( + """CREATE TABLE IF NOT EXISTS receipt_schema ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), + version INTEGER NOT NULL CHECK(version > 0) + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS delivery_targets ( + id INTEGER PRIMARY KEY, + execution_id TEXT NOT NULL REFERENCES executions(id) ON DELETE CASCADE, + platform TEXT NOT NULL CHECK(length(platform) BETWEEN 1 AND 64), + chat_id TEXT NOT NULL CHECK(length(chat_id) BETWEEN 1 AND 512), + thread_id TEXT NOT NULL DEFAULT '' CHECK(length(thread_id) <= 512), + UNIQUE(execution_id, platform, chat_id, thread_id) + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS delivery_components ( + id INTEGER PRIMARY KEY, + target_id INTEGER NOT NULL REFERENCES delivery_targets(id) ON DELETE CASCADE, + component TEXT NOT NULL CHECK(length(component) BETWEEN 1 AND 64), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + content_hash TEXT NOT NULL CHECK(length(content_hash) = 64), + UNIQUE(target_id, component, ordinal, content_hash) + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS delivery_attempts ( + id TEXT PRIMARY KEY, + component_id INTEGER NOT NULL REFERENCES delivery_components(id) ON DELETE CASCADE, + attempt_no INTEGER NOT NULL CHECK(attempt_no >= 1), + outcome TEXT NOT NULL CHECK(outcome IN ('delivered','unknown','failed')), + provider_message_id TEXT CHECK(provider_message_id IS NULL OR length(provider_message_id) BETWEEN 1 AND 1024), + actual_platform TEXT CHECK(actual_platform IS NULL OR length(actual_platform) BETWEEN 1 AND 64), + actual_chat_id TEXT CHECK(actual_chat_id IS NULL OR length(actual_chat_id) BETWEEN 1 AND 512), + actual_thread_id TEXT CHECK(actual_thread_id IS NULL OR length(actual_thread_id) <= 512), + failure_kind TEXT CHECK(failure_kind IS NULL OR length(failure_kind) BETWEEN 1 AND 64), + observed_at TEXT, + created_at TEXT NOT NULL, + UNIQUE(component_id, attempt_no) + )""" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_delivery_attempts_component " + "ON delivery_attempts(component_id, attempt_no)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_delivery_targets_execution " + "ON delivery_targets(execution_id, id)" + ) + # Legacy executions have no provider-target evidence. Do not manufacture + # fake targets/attempt ids: receipt_summary() projects them as one bounded + # attempted_unconfirmed/unknown result at read time instead. + conn.execute( + """INSERT INTO receipt_schema(singleton, version) VALUES (1, ?) + ON CONFLICT(singleton) DO UPDATE SET version=excluded.version""", + (_RECEIPT_SCHEMA_VERSION,), + ) + conn.commit() @contextmanager @@ -151,19 +258,422 @@ def _prune_unlocked(conn: sqlite3.Connection) -> None: ) -def create_execution(job_id: str, *, source: str) -> Dict[str, Any]: - """Persist a claimed attempt before executor/provider dispatch.""" +def _bounded_text(value: Any, *, field: str, limit: int, allow_empty: bool = False) -> str: + if value is None and allow_empty: + return "" + if type(value) is not str: + raise ValueError(f"{field} must be a string") + text = unicodedata.normalize("NFC", value) + if ( + (not allow_empty and not text) + or len(text) > limit + or text != value + or any(char.isspace() or unicodedata.category(char).startswith("C") for char in text) + ): + raise ValueError(f"{field} must be {'non-empty and ' if not allow_empty else ''}at most {limit} characters") + return text + + +def _component_hash(content: Any) -> str: + """Hash canonical logical text without placing the content in SQLite.""" + if type(content) is not str: + raise ValueError("content must be a string") + canonical = unicodedata.normalize("NFC", content) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def preregister_receipt_plan( + execution_id: str, + *, + fire_identity: str, + components: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Atomically create unknown receipt attempts before any external dispatch. + + ``components`` is intentionally a tiny internal plan shape. It accepts + logical content only long enough to calculate its SHA-256; no body, media + path, raw provider response, exception, or credential reaches the DB. + Callers must abort dispatch if this raises. + """ + _bounded_text(execution_id, field="execution_id", limit=128) + _bounded_text(fire_identity, field="fire_identity", limit=256) + if type(components) is not list: + raise ValueError("components must be a list") + prepared = [] + for item in components: + if type(item) is not dict: + raise ValueError("every receipt component must be an object") + target_value = item.get("target") + if type(target_value) is not dict: + raise ValueError("every receipt component requires a target") + target = target_value + ordinal = item.get("ordinal", 0) + if type(ordinal) is not int or ordinal < 0: + raise ValueError("component ordinal must be a non-negative integer") + prepared.append( + ( + _bounded_text(target.get("platform"), field="platform", limit=64), + _bounded_text(target.get("chat_id"), field="chat_id", limit=512), + _bounded_text(target.get("thread_id"), field="thread_id", limit=512, allow_empty=True), + _bounded_text(item.get("component"), field="component", limit=64), + ordinal, + _component_hash(item.get("content", "")), + ) + ) + if not prepared: + return [] + + + created_at = _hermes_now().isoformat() + created: List[Dict[str, Any]] = [] + with _transaction() as conn: + execution = conn.execute( + "SELECT job_id, fire_identity, receipt_state FROM executions WHERE id=?", (execution_id,) + ).fetchone() + if execution is None: + raise ValueError("unknown execution_id") + existing_fire = execution["fire_identity"] + if existing_fire is None: + conn.execute( + """UPDATE executions SET fire_identity=?, receipt_state='planned' + WHERE id=? AND fire_identity IS NULL""", + (fire_identity, execution_id), + ) + elif existing_fire != fire_identity: + if existing_fire == execution_id and execution["receipt_state"] == "not_planned": + conn.execute( + "UPDATE executions SET fire_identity=? WHERE id=?", + (fire_identity, execution_id), + ) + else: + raise ValueError("conflicting receipt plan fire_identity") + existing_rows = conn.execute( + """SELECT a.id, a.outcome, c.component, c.ordinal, c.content_hash, + t.platform, t.chat_id, t.thread_id + FROM delivery_attempts a + JOIN delivery_components c ON c.id=a.component_id + JOIN delivery_targets t ON t.id=c.target_id + WHERE t.execution_id=? AND a.attempt_no=1 + ORDER BY t.id, c.component, c.ordinal""", + (execution_id,), + ).fetchall() + if existing_rows: + existing_plan = { + (row["platform"], row["chat_id"], row["thread_id"], row["component"], + row["ordinal"], row["content_hash"]) + for row in existing_rows + } + if existing_plan != set(prepared) or len(existing_rows) != len(prepared): + raise ValueError("conflicting receipt plan") + return [ + {"id": row["id"], "execution_id": execution_id, + "platform": row["platform"], "chat_id": row["chat_id"], + "thread_id": row["thread_id"], "component": row["component"], + "ordinal": row["ordinal"], "outcome": row["outcome"]} + for row in existing_rows + ] + # A fire identity is single-dispatch. Once any component was + # preregistered, an unknown outcome cannot authorize replay: the + # provider write may have completed before acknowledgement was lost. + # A different fire identity remains independent, even for identical + # content. + cutoff = (_hermes_now() - timedelta(hours=24)).isoformat() + prior_rows = conn.execute( + """SELECT t.platform, t.chat_id, t.thread_id, + c.component, c.ordinal, c.content_hash + FROM delivery_attempts a + JOIN delivery_components c ON c.id=a.component_id + JOIN delivery_targets t ON t.id=c.target_id + JOIN executions e ON e.id=t.execution_id + WHERE e.id<>? AND e.job_id=? AND e.fire_identity=? + AND e.claimed_at>=? + ORDER BY t.platform, t.chat_id, t.thread_id, c.component, c.ordinal""", + (execution_id, execution["job_id"], fire_identity, cutoff), + ).fetchall() + if prior_rows: + prior_plan = { + ( + row["platform"], row["chat_id"], row["thread_id"], + row["component"], row["ordinal"], row["content_hash"], + ) + for row in prior_rows + } + if prior_plan != set(prepared) or len(prior_rows) != len(prepared): + raise ValueError("conflicting receipt plan for existing fire_identity") + raise ValueError("delivery receipt already attempted for this fire identity") + for platform, chat_id, thread_id, component, ordinal, content_hash in prepared: + conn.execute( + """INSERT OR IGNORE INTO delivery_targets + (execution_id, platform, chat_id, thread_id) VALUES (?, ?, ?, ?)""", + (execution_id, platform, chat_id, thread_id), + ) + target_row = conn.execute( + """SELECT id FROM delivery_targets WHERE execution_id=? AND platform=? + AND chat_id=? AND thread_id=?""", + (execution_id, platform, chat_id, thread_id), + ).fetchone() + assert target_row is not None + component_row = conn.execute( + """INSERT INTO delivery_components(target_id, component, ordinal, content_hash) + VALUES (?, ?, ?, ?) RETURNING id""", + (target_row["id"], component, ordinal, content_hash), + ).fetchone() + attempt_id = uuid.uuid4().hex + conn.execute( + """INSERT INTO delivery_attempts + (id, component_id, attempt_no, outcome, created_at) + VALUES (?, ?, 1, 'unknown', ?)""", + (attempt_id, component_row["id"], created_at), + ) + created.append({ + "id": attempt_id, + "execution_id": execution_id, + "platform": platform, + "chat_id": chat_id, + "thread_id": thread_id, + "component": component, + "ordinal": ordinal, + "outcome": "unknown", + }) + return created + + +def receipt_summary(execution_id: str) -> Dict[str, int]: + """Return bounded receipt counts; never expose component content or errors.""" + with _transaction() as conn: + counts = {"delivered": 0, "failed": 0, "unknown": 0, "targets_delivered": 0} + for row in conn.execute( + """SELECT outcome, COUNT(*) AS count FROM delivery_attempts a + JOIN delivery_components c ON c.id=a.component_id + JOIN delivery_targets t ON t.id=c.target_id + WHERE t.execution_id=? GROUP BY outcome""", + (execution_id,), + ): + counts[row["outcome"]] = int(row["count"]) + target_row = conn.execute( + """SELECT e.receipt_state, COUNT(t.id) AS count + FROM executions e LEFT JOIN delivery_targets t ON t.execution_id=e.id + WHERE e.id=? GROUP BY e.id""", + (execution_id,), + ).fetchone() + if ( + target_row is not None + and int(target_row["count"]) == 0 + and target_row["receipt_state"] is None + ): + # A pre-receipt/legacy execution has an attempted side effect but + # no provider proof. Surface that honestly without creating rows + # during a read operation. + counts["unknown"] = 1 + target_row = conn.execute( + """SELECT COUNT(*) AS count FROM ( + SELECT t.id FROM delivery_targets t + JOIN delivery_components c ON c.target_id=t.id + JOIN delivery_attempts a ON a.component_id=c.id + WHERE t.execution_id=? + GROUP BY t.id + HAVING COUNT(DISTINCT c.id) = COUNT(DISTINCT CASE + WHEN a.outcome='delivered' + AND a.actual_platform=t.platform + AND a.actual_chat_id=t.chat_id + AND COALESCE(a.actual_thread_id, '')=t.thread_id + THEN c.id END) + )""", + (execution_id,), + ).fetchone() + counts["targets_delivered"] = int(target_row["count"] if target_row else 0) + return counts + + +def record_transport_receipt(attempt_id: str, receipt: Any) -> bool: + """Commit a typed provider acknowledgement once, preserving unknown on error. + + This function intentionally has no retry behavior. If the caller receives + an acknowledgement but this transaction fails, the attempt remains unknown + and the caller must not infer that a resend is safe. + """ + from gateway.platforms.base import TransportReceipt, TransportTarget + + if type(receipt) is not TransportReceipt: + raise ValueError("receipt must be a TransportReceipt") + if type(receipt.ordinal) is not int or receipt.ordinal < 0: + raise ValueError("receipt ordinal must be a non-negative integer") + requested = receipt.requested_target + actual = receipt.actual_target + if type(requested) is not TransportTarget: + raise TypeError("requested_target must be a TransportTarget") + if actual is not None and type(actual) is not TransportTarget: + raise TypeError("actual_target must be a TransportTarget") + + # ``frozen=True`` emulates immutability but can still be bypassed with + # ``object.__setattr__``. Reconstruct the complete typed contract at this + # persistence boundary so every target and receipt invariant is re-run + # immediately before durable state can change. Keep the explicit outcome + # checks above first so their established error categories remain stable. + requested = TransportTarget( + platform=requested.platform, + chat_id=requested.chat_id, + thread_id=requested.thread_id, + ) + if actual is not None: + actual = TransportTarget( + platform=actual.platform, + chat_id=actual.chat_id, + thread_id=actual.thread_id, + ) + receipt = TransportReceipt( + outcome=receipt.outcome, + requested_target=requested, + actual_target=actual, + provider_message_id=receipt.provider_message_id, + observed_at=receipt.observed_at, + failure_kind=receipt.failure_kind, + component=receipt.component, + ordinal=receipt.ordinal, + ) + outcome = receipt.outcome + if outcome not in {"delivered", "failed"}: + return False + provider_message_id = receipt.provider_message_id + failure_kind = receipt.failure_kind + actual_platform = getattr(actual, "platform", None) if actual is not None else None + actual_chat_id = getattr(actual, "chat_id", None) if actual is not None else None + actual_thread_id = getattr(actual, "thread_id", None) if actual is not None else None + if provider_message_id is not None: + provider_message_id = _bounded_text(provider_message_id, field="provider_message_id", limit=1024) + if actual_platform is not None: + actual_platform = _bounded_text(actual_platform, field="actual_platform", limit=64) + actual_chat_id = _bounded_text(actual_chat_id, field="actual_chat_id", limit=512) + actual_thread_id = _bounded_text(actual_thread_id, field="actual_thread_id", limit=512, allow_empty=True) or None + if failure_kind is not None: + failure_kind = _bounded_text(failure_kind, field="failure_kind", limit=64) + observed_at = receipt.observed_at + observed_text = observed_at.isoformat() + normalized_attempt_id = _bounded_text( + attempt_id, field="attempt_id", limit=128, + ) + with _transaction() as conn: + binding = conn.execute( + """SELECT c.component, c.ordinal, t.platform, t.chat_id, t.thread_id + FROM delivery_attempts a + JOIN delivery_components c ON c.id=a.component_id + JOIN delivery_targets t ON t.id=c.target_id WHERE a.id=?""", + (normalized_attempt_id,), + ).fetchone() + if binding is None: + return False + if ( + requested.platform != binding["platform"] + or requested.chat_id != binding["chat_id"] + or (requested.thread_id or "") != binding["thread_id"] + or receipt.component != binding["component"] + or receipt.ordinal != binding["ordinal"] + ): + raise ValueError("receipt requested_target/component does not match preregistered attempt") + cur = conn.execute( + """UPDATE delivery_attempts SET outcome=?, provider_message_id=?, + actual_platform=?, actual_chat_id=?, actual_thread_id=?, + failure_kind=?, observed_at=? + WHERE id=? AND outcome='unknown'""", + ( + outcome, + provider_message_id, + actual_platform, + actual_chat_id, + actual_thread_id, + failure_kind, + observed_text, + normalized_attempt_id, + ), + ) + return cur.rowcount == 1 + + +def observe_transport_unknown(attempt_id: str, receipt: Any) -> bool: + """Record a typed ambiguous observation without upgrading its outcome.""" + from gateway.platforms.base import TransportReceipt, TransportTarget + + if type(receipt) is not TransportReceipt: + raise ValueError("receipt must be a typed unknown TransportReceipt") + requested = receipt.requested_target + actual = receipt.actual_target + if type(requested) is not TransportTarget: + raise TypeError("requested_target must be a TransportTarget") + if actual is not None and type(actual) is not TransportTarget: + raise TypeError("actual_target must be a TransportTarget") + # Reconstruct at the persistence boundary so frozen-dataclass bypasses and + # malformed target fields cannot reach SQLite. + requested = TransportTarget( + platform=requested.platform, + chat_id=requested.chat_id, + thread_id=requested.thread_id, + ) + receipt = TransportReceipt( + outcome=receipt.outcome, + requested_target=requested, + actual_target=actual, + provider_message_id=receipt.provider_message_id, + observed_at=receipt.observed_at, + failure_kind=receipt.failure_kind, + component=receipt.component, + ordinal=receipt.ordinal, + ) + if receipt.outcome != "unknown": + raise ValueError("receipt must be a typed unknown TransportReceipt") + observed_text = receipt.observed_at.isoformat() + normalized_attempt_id = _bounded_text( + attempt_id, field="attempt_id", limit=128, + ) + with _transaction() as conn: + binding = conn.execute( + """SELECT c.component, c.ordinal, t.platform, t.chat_id, t.thread_id + FROM delivery_attempts a + JOIN delivery_components c ON c.id=a.component_id + JOIN delivery_targets t ON t.id=c.target_id WHERE a.id=?""", + (normalized_attempt_id,), + ).fetchone() + if binding is None: + return False + if ( + requested.platform != binding["platform"] + or requested.chat_id != binding["chat_id"] + or (requested.thread_id or "") != binding["thread_id"] + or receipt.component != binding["component"] + or receipt.ordinal != binding["ordinal"] + ): + raise ValueError( + "receipt requested_target/component does not match preregistered attempt" + ) + cur = conn.execute( + """UPDATE delivery_attempts SET observed_at=? + WHERE id=? AND outcome='unknown' AND observed_at IS NULL""", + (observed_text, normalized_attempt_id), + ) + return cur.rowcount == 1 + + +def create_execution( + job_id: str, *, source: str, fire_identity: Optional[str] = None, +) -> Dict[str, Any]: + """Persist a claimed attempt and its fire identity before dispatch.""" + normalized_job_id = _bounded_text(job_id, field="job_id", limit=256) + normalized_source = _bounded_text(source, field="source", limit=32) now = _hermes_now().isoformat() execution_id = uuid.uuid4().hex + normalized_fire_identity = _bounded_text( + execution_id if fire_identity is None else fire_identity, + field="fire_identity", + limit=256, + ) pid = os.getpid() with _transaction() as conn: conn.execute( """INSERT INTO executions (id, job_id, source, process_id, pid, process_started_at, - status, claimed_at) - VALUES (?, ?, ?, ?, ?, ?, 'claimed', ?)""", - (execution_id, str(job_id), str(source), _PROCESS_ID, pid, - _process_start_time(pid), now), + status, claimed_at, receipt_state, fire_identity) + VALUES (?, ?, ?, ?, ?, ?, 'claimed', ?, 'not_planned', ?)""", + (execution_id, normalized_job_id, normalized_source, _PROCESS_ID, pid, + _process_start_time(pid), now, normalized_fire_identity), ) row = conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) @@ -173,6 +683,40 @@ def create_execution(job_id: str, *, source: str) -> Dict[str, Any]: return record # type: ignore[return-value] +def bind_execution_fire_identity( + execution_id: str, fire_identity: str, +) -> Dict[str, Any]: + """Bind a claimed execution to its acquired fire exactly once before planning.""" + normalized_execution_id = _bounded_text( + execution_id, field="execution_id", limit=128, + ) + normalized_fire_identity = _bounded_text( + fire_identity, field="fire_identity", limit=256, + ) + with _transaction() as conn: + row = conn.execute( + "SELECT * FROM executions WHERE id=?", (normalized_execution_id,), + ).fetchone() + if row is None: + raise ValueError("unknown execution_id") + if row["fire_identity"] == normalized_fire_identity: + return _record(row) # type: ignore[return-value] + if ( + row["status"] != "claimed" + or row["receipt_state"] != "not_planned" + or row["fire_identity"] != normalized_execution_id + ): + raise ValueError("execution fire_identity is already bound") + conn.execute( + "UPDATE executions SET fire_identity=? WHERE id=?", + (normalized_fire_identity, normalized_execution_id), + ) + bound = conn.execute( + "SELECT * FROM executions WHERE id=?", (normalized_execution_id,), + ).fetchone() + return _record(bound) # type: ignore[return-value] + + def mark_execution_handoff_pending(execution_id: str) -> Optional[Dict[str, Any]]: """Fence restart recovery while an external worker is adopting a claim.""" with _transaction() as conn: @@ -243,20 +787,24 @@ def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]: def finish_execution( execution_id: str, *, success: bool, error: Optional[str] = None, + error_kind: Optional[str] = None, delivery_outcome: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """Write a terminal result once; terminal attempts cannot be rewritten.""" now = _hermes_now().isoformat() status = "completed" if success else "failed" - detail = None if success else (str(error) if error else "unknown failure") + del error # Free-form diagnostics must never enter the durable ledger. + category = None if success else (error_kind or "execution_failed") + if category is not None and category not in _EXECUTION_ERROR_KINDS: + raise ValueError("execution error_kind is invalid") with _transaction() as conn: cur = conn.execute( """UPDATE executions - SET status=?, finished_at=?, error=?, handoff_pending=0, + SET status=?, finished_at=?, error=NULL, error_kind=?, handoff_pending=0, handoff_started_at=NULL WHERE id=? AND status IN ('claimed','running') AND process_id=? AND pid=?""", - (status, now, detail, execution_id, _PROCESS_ID, os.getpid()), + (status, now, category, execution_id, _PROCESS_ID, os.getpid()), ) if cur.rowcount != 1: return None @@ -295,15 +843,13 @@ def recover_interrupted_executions() -> int: continue cur = conn.execute( """UPDATE executions - SET status='unknown', finished_at=?, error=?, + SET status='unknown', finished_at=?, error=NULL, + error_kind='interrupted', handoff_pending=0, handoff_started_at=NULL WHERE id=? AND status=? AND process_id=? AND pid=? AND handoff_pending=? AND handoff_started_at IS ?""", - (now, - "Scheduler restarted after this execution's owner exited before a durable " - "terminal state; whether side effects ran is unknown.", - row["id"], row["status"], row["process_id"], row["pid"], + (now, row["id"], row["status"], row["process_id"], row["pid"], row["handoff_pending"], row["handoff_started_at"]), ) changed += cur.rowcount @@ -329,12 +875,16 @@ def list_executions( params: List[Any] = [] if job_id is not None: clauses.append("job_id=?") - params.append(str(job_id)) + params.append(_bounded_text(job_id, field="job_id", limit=256)) if before_claimed_at is not None: clauses.append("claimed_at < ?") - params.append(str(before_claimed_at)) + params.append(_bounded_text( + before_claimed_at, field="before_claimed_at", limit=128, + )) where = " WHERE " + " AND ".join(clauses) if clauses else "" - params.append(max(1, min(int(limit), 500))) + if type(limit) is not int: + raise ValueError("limit must be an integer") + params.append(max(1, min(limit, 500))) with _transaction() as conn: rows = conn.execute( "SELECT * FROM executions" + where @@ -361,7 +911,15 @@ def latest_execution(job_id: str) -> Optional[Dict[str, Any]]: def latest_executions(job_ids: List[str]) -> Dict[str, Dict[str, Any]]: """Load latest execution for many jobs in one indexed query.""" - clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id] + if type(job_ids) is not list: + raise ValueError("job_ids must be a list") + clean = [] + seen = set() + for job_id in job_ids: + normalized = _bounded_text(job_id, field="job_id", limit=256) + if normalized not in seen: + seen.add(normalized) + clean.append(normalized) if not clean: return {} placeholders = ",".join("?" for _ in clean) diff --git a/cron/jobs.py b/cron/jobs.py index 385d747619e8..2d2f94eab6fa 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -3665,6 +3665,7 @@ def _claim_job_for_fire_locked( return False now = _hermes_now() existing = job.get("fire_claim") + fire_at = now.isoformat() if existing: try: claimed_at = _ensure_aware(datetime.fromisoformat(existing["at"])) @@ -3679,6 +3680,10 @@ def _claim_job_for_fire_locked( return False # someone holds a fresh claim except Exception: pass # malformed claim → overwrite + if not force: + existing_fire_at = existing.get("fire_at") + if isinstance(existing_fire_at, str) and existing_fire_at: + fire_at = existing_fire_at if force: job["enabled"] = True job["state"] = "scheduled" @@ -3688,7 +3693,11 @@ def _claim_job_for_fire_locked( # stale lease, and the previous runner must not heartbeat the new # claim merely because hostname + PID are unchanged. owner = f"{_machine_id()}:{uuid.uuid4().hex}" - job["fire_claim"] = {"at": now.isoformat(), "by": owner} + job["fire_claim"] = { + "at": now.isoformat(), + "fire_at": fire_at, + "by": owner, + } kind = job.get("schedule", {}).get("kind") if kind in {"cron", "interval"}: nxt = compute_next_run(job["schedule"], now.isoformat()) diff --git a/cron/scheduler.py b/cron/scheduler.py index 267805a8f1a9..52a1c555c89c 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -26,6 +26,7 @@ import time import uuid from datetime import datetime, timezone +from types import SimpleNamespace # fcntl is Unix-only; on Windows use msvcrt for file locking try: @@ -727,12 +728,18 @@ def _resolve_job_reasoning_config(job: dict, cfg: dict, model: str) -> dict | No ) from cron.executions import ( _TERMINAL_STATES, + bind_execution_fire_identity, create_execution, finish_execution, get_execution, mark_execution_handoff_pending, mark_execution_running, + observe_transport_unknown, + preregister_receipt_plan, recover_interrupted_executions, + record_transport_receipt, + receipt_summary, + scheduled_fire_identity, ) # Sentinel: when a cron agent has nothing new to report, it can start its @@ -1668,11 +1675,18 @@ def _resolve_origin(job: dict) -> Optional[dict]: crashed identically until the field was patched manually (#18722). """ origin = job.get("origin") - if not isinstance(origin, dict): + if type(origin) is not dict: return None platform = origin.get("platform") chat_id = origin.get("chat_id") - if platform and chat_id: + if platform is None or chat_id is None: + return None + if type(platform) is not str or type(chat_id) not in {str, int}: + raise ValueError("origin target identity is invalid") + thread_id = origin.get("thread_id") + if thread_id is not None and type(thread_id) not in {str, int}: + raise ValueError("origin target identity is invalid") + if platform and chat_id not in {"", 0}: return origin return None @@ -2642,6 +2656,16 @@ def _get_bot_chat_delivery_timeout() -> int: return 600 +def _bot_chat_query_message(job: dict, content: str) -> str: + """Compose the exact child query bytes for planning and dispatch.""" + job_name = job.get("name", job.get("id", "?")) + return ( + f'[Cronjob "{job_name}" output — scheduled job, not the user. ' + f"Review it, act on anything that needs action, and summarize " + f"for the chat.]\n\n{content}" + ) + + def _deliver_to_bot_chat(job: dict, content: str, profile: str) -> Optional[str]: """Deliver job output into a profile's canonical Bot Chat as an inbound turn. @@ -2660,7 +2684,6 @@ def _deliver_to_bot_chat(job: dict, content: str, profile: str) -> Optional[str] import tempfile job_id = job.get("id", "?") - job_name = job.get("name", job_id) hermes_bin = _shutil.which("hermes") if hermes_bin: @@ -2672,9 +2695,9 @@ def _deliver_to_bot_chat(job: dict, content: str, profile: str) -> Optional[str] if _ilu.find_spec("hermes_cli") is not None: argv = [sys.executable, "-m", "hermes_cli.main"] else: - return "bot-chat delivery failed: hermes CLI not resolvable" + return "bot-chat delivery failed" except Exception: - return "bot-chat delivery failed: hermes CLI not resolvable" + return "bot-chat delivery failed" env = os.environ.copy() if profile: @@ -2684,12 +2707,8 @@ def _deliver_to_bot_chat(job: dict, content: str, profile: str) -> Optional[str] env.pop("HERMES_HOME", None) # The prefix tells the receiving bot this is scheduled output, not the - # human typing — mirrors the Bot Mode sender-attribution convention. - message = ( - f'[Cronjob "{job_name}" output — scheduled job, not the user. ' - f"Review it, act on anything that needs action, and summarize " - f"for the chat.]\n\n{content}" - ) + # human typing. Planning calls the same helper before any side effect. + message = _bot_chat_query_message(job, content) query_file = None try: @@ -2714,32 +2733,18 @@ def _deliver_to_bot_chat(job: dict, content: str, profile: str) -> Optional[str] creationflags=windows_hide_flags(), ) if result.returncode != 0: - tail = (result.stderr or result.stdout or "").strip()[-500:] - msg = ( - f"bot-chat delivery to profile " - f"'{profile or '(own)'}' failed (exit {result.returncode})" - + (f": {tail}" if tail else "") - ) - logger.warning("Job '%s': %s", job_id, msg) - return msg + logger.warning("Job '%s': bot-chat delivery confirmation unavailable", job_id) + return "bot-chat delivery confirmation unavailable" logger.info( - "Job '%s': delivered to Bot Chat of profile '%s'", - job_id, profile or "(own)", + "Job '%s': bot-chat child completed without provider receipt", job_id ) return None except subprocess.TimeoutExpired: - msg = ( - f"bot-chat delivery to profile '{profile or '(own)'}' timed out " - f"after {_get_bot_chat_delivery_timeout()}s (the bot's turn may " - "still complete; raise cron.bot_chat_delivery_timeout_seconds if " - "this recurs)" - ) - logger.warning("Job '%s': %s", job_id, msg) - return msg - except Exception as e: - msg = f"bot-chat delivery failed: {str(e) or type(e).__name__}" - logger.warning("Job '%s': %s", job_id, msg, exc_info=True) - return msg + logger.warning("Job '%s': bot-chat delivery confirmation unavailable", job_id) + return "bot-chat delivery confirmation unavailable" + except Exception: + logger.warning("Job '%s': bot-chat delivery confirmation unavailable", job_id) + return "bot-chat delivery confirmation unavailable" finally: if query_file: try: @@ -2760,12 +2765,44 @@ def _normalize_deliver_value(deliver) -> str: resolution silently. Flatten lists/tuples into a comma-separated string so both forms work. Returns ``"local"`` for anything falsy. """ - if deliver is None or deliver == "": + if deliver is None: return "local" - if isinstance(deliver, (list, tuple)): - parts = [str(p).strip() for p in deliver if str(p).strip()] + if type(deliver) is str: + return deliver or "local" + if type(deliver) in {list, tuple}: + parts = [p.strip() for p in deliver if type(p) is str and p.strip()] return ",".join(parts) if parts else "local" - return str(deliver) + return "local" + + +def _normalize_delivery_target_identity(target: Any) -> dict: + """Return a content-free target using only inert built-in scalar values.""" + if type(target) is not dict: + raise ValueError("delivery target must be an object") + platform = target.get("platform") + chat_id = target.get("chat_id") + thread_id = target.get("thread_id") + if type(platform) is not str or not platform: + raise ValueError("delivery target platform is invalid") + if type(chat_id) not in {str, int} or chat_id in {"", 0}: + raise ValueError("delivery target chat_id is invalid") + if thread_id is not None and ( + type(thread_id) not in {str, int} or thread_id in {"", 0} + ): + raise ValueError("delivery target thread_id is invalid") + normalized = { + "platform": platform, + "chat_id": str(chat_id), + "thread_id": str(thread_id) if thread_id is not None else None, + } + resolved_from = target.get("_resolved_from") + if resolved_from is not None: + if type(resolved_from) is not str or resolved_from not in { + "origin", "origin_fallback", "explicit", + }: + raise ValueError("delivery target provenance is invalid") + normalized["_resolved_from"] = resolved_from + return normalized # Routing intent tokens — resolved at fire time, not create time, so a @@ -2782,6 +2819,7 @@ def _normalize_deliver_value(deliver) -> str: # ``all`` routing token: ``all`` fans out to messaging home channels, and a # bot-chat delivery costs a full agent turn. BOT_CHAT_PLATFORM = "bot-chat" +BOT_CHAT_SELF_TARGET = "_self" def parse_bot_chat_deliver_token(part: str) -> Optional[str]: @@ -2814,8 +2852,13 @@ def _resolve_bot_chat_target(job: dict, profile_arg: str) -> Optional[dict]: can never be targeted by accident. """ if not profile_arg: - # Own profile: chat subprocess inherits HERMES_HOME, no name needed. - return {"platform": BOT_CHAT_PLATFORM, "chat_id": "", "thread_id": None} + # Own profile: the child inherits HERMES_HOME; the ledger still needs a + # concrete non-empty requested-target identity. + return { + "platform": BOT_CHAT_PLATFORM, + "chat_id": BOT_CHAT_SELF_TARGET, + "thread_id": None, + } try: from hermes_cli.profiles import normalize_profile_name, profile_exists @@ -2901,8 +2944,9 @@ def _resolve_delivery_targets(job: dict, *, for_failure: bool = False) -> List[d targets = [] for part in parts: target = _resolve_single_delivery_target(job, part) - if target: - key = (target["platform"].lower(), str(target["chat_id"]), target.get("thread_id")) + if target is not None: + target = _normalize_delivery_target_identity(target) + key = (target["platform"].lower(), target["chat_id"], target["thread_id"]) if key not in seen: seen[key] = target targets.append(target) @@ -2938,6 +2982,7 @@ def _send_media_via_adapter( loop, job: dict, platform=None, + receipts_out: Optional[list] = None, ) -> list: """Send extracted MEDIA files as native platform attachments via a live adapter. @@ -2953,7 +2998,11 @@ def _send_media_via_adapter( """ from pathlib import Path - from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio + from gateway.platforms.base import ( + BasePlatformAdapter, + SendResult, + should_send_media_as_audio, + ) errors: list = [] requested = [(str(p), v) for p, v in (media_files or [])] @@ -2973,18 +3022,21 @@ def _send_media_via_adapter( except Exception: errors.append(f"attachment dropped by media path policy: {raw_path}") - for media_path, _is_voice in media_files: + for media_ordinal, (media_path, _is_voice) in enumerate(media_files): try: + send_metadata = dict(metadata or {}) + send_metadata["_transport_receipt_component"] = "media" + send_metadata["_transport_receipt_ordinal"] = media_ordinal ext = Path(media_path).suffix.lower() route_platform = platform if platform is not None else getattr(adapter, "platform", None) if should_send_media_as_audio(route_platform, ext, is_voice=_is_voice): - coro = adapter.send_voice(chat_id=chat_id, audio_path=media_path, metadata=metadata) + coro = adapter.send_voice(chat_id=chat_id, audio_path=media_path, metadata=send_metadata) elif ext in _VIDEO_EXTS: - coro = adapter.send_video(chat_id=chat_id, video_path=media_path, metadata=metadata) + coro = adapter.send_video(chat_id=chat_id, video_path=media_path, metadata=send_metadata) elif ext in _IMAGE_EXTS: - coro = adapter.send_image_file(chat_id=chat_id, image_path=media_path, metadata=metadata) + coro = adapter.send_image_file(chat_id=chat_id, image_path=media_path, metadata=send_metadata) else: - coro = adapter.send_document(chat_id=chat_id, file_path=media_path, metadata=metadata) + coro = adapter.send_document(chat_id=chat_id, file_path=media_path, metadata=send_metadata) from agent.async_utils import safe_schedule_threadsafe future = safe_schedule_threadsafe(coro, loop) @@ -3003,10 +3055,26 @@ def _send_media_via_adapter( except TimeoutError: future.cancel() raise - if result and not getattr(result, "success", True): + receipt_bound = ( + type(metadata) is dict + and "_transport_receipt_requested_target" in metadata + ) + if type(result) is SendResult: + result_success = result.success is True + result_error = result.error + if receipts_out is not None: + receipts_out.extend(result.receipts) + else: + legacy = _inert_legacy_send_result_fields(result) + if receipt_bound or legacy is None: + errors.append("media adapter returned an invalid result") + return errors + result_success = legacy["success"] is True + result_error = legacy["error"] + if not result_success: msg = ( f"media send failed for {media_path}: " - f"{getattr(result, 'error', 'unknown')}" + f"{result_error or 'unknown'}" ) logger.warning("Job '%s': %s", job.get("id", "?"), msg) errors.append(msg) @@ -3051,23 +3119,34 @@ def _confirm_adapter_delivery(send_result, job_id: str = "?", unverified: Option instead of masquerading as a confirmed one. Telegram ``SendResult`` objects carry ``message_id``; the dict-filter shape does not. """ - if send_result is None: - return False - if isinstance(send_result, dict): - if "success" not in send_result: + from gateway.platforms.base import SendResult + + if type(send_result) is dict: + if type(send_result.get("success")) is not bool: return False - success = bool(send_result.get("success")) + success = send_result["success"] delivered = send_result.get("delivered") message_id = send_result.get("message_id") - raw_response = send_result.get("raw_response") - else: - if not hasattr(send_result, "success"): + raw_response = ( + send_result.get("raw_response") + if type(send_result.get("raw_response")) is dict else None + ) + elif type(send_result) is SendResult: + if type(send_result.success) is not bool: return False - success = bool(getattr(send_result, "success")) + success = send_result.success delivered = getattr(send_result, "delivered", None) - message_id = getattr(send_result, "message_id", None) - raw_response = getattr(send_result, "raw_response", None) - if not success or delivered is False: + message_id = send_result.message_id + raw_response = send_result.raw_response + else: + legacy = _inert_legacy_send_result_fields(send_result) + if legacy is None: + return False + success = legacy["success"] + delivered = None + message_id = legacy["message_id"] + raw_response = legacy["raw_response"] + if success is not True or delivered is False: return False if message_id is None and not raw_response: logger.warning( @@ -3081,6 +3160,27 @@ def _confirm_adapter_delivery(send_result, job_id: str = "?", unverified: Option return True +def _inert_legacy_send_result_fields(send_result: Any) -> Optional[dict]: + """Read the one supported inert legacy container without object magic.""" + if type(send_result) is not SimpleNamespace: + return None + fields = object.__getattribute__(send_result, "__dict__") + if type(fields) is not dict or type(fields.get("success")) is not bool: + return None + error = fields.get("error") + message_id = fields.get("message_id") + if error is not None and type(error) is not str: + error = None + if message_id is not None and type(message_id) not in {str, int}: + message_id = None + return { + "success": fields["success"], + "error": error, + "message_id": message_id, + "raw_response": fields.get("raw_response") if type(fields.get("raw_response")) is dict else None, + } + + def _is_channel_dm_topic( runtime_adapter: Any, chat_id: Any, @@ -3137,6 +3237,151 @@ def _is_channel_dm_topic( return is_channel +def _receipt_text_chunks_for_target( + adapters: Any, platform_name: str, content: str, media_files=None, +) -> list[str]: + """Return exact adapter-planned chunks when the adapter can prove them. + + Opaque and standalone transports deliberately get one logical component: + a later multi-ack cannot be upgraded to delivery unless it binds that plan. + Matrix and Telegram expose this preflight because their send paths own the + deterministic formatting/splitting algorithm. + """ + if not adapters: + if platform_name.lower() == "telegram": + from tools.send_message_tool import _plan_standalone_telegram_text + + return _plan_standalone_telegram_text( + content, media_files=media_files, + )[1] + return [content] + candidate = None + try: + from gateway.config import Platform + candidate = adapters.get(Platform(platform_name.lower())) + except Exception: + candidate = None + if candidate is None: + try: + candidate = adapters.get(platform_name) or adapters.get(platform_name.lower()) + except Exception: + candidate = None + if candidate is None: + if platform_name.lower() == "telegram": + from tools.send_message_tool import _plan_standalone_telegram_text + + return _plan_standalone_telegram_text( + content, media_files=media_files, + )[1] + return [content] + planner = getattr(candidate, "plan_transport_text", None) + if not callable(planner): + return [content] + try: + chunks = planner(content) + except Exception as exc: + raise ValueError("transport planner failed before dispatch") from exc + if ( + type(chunks) not in {list, tuple} + or len(chunks) == 0 + or not all(type(chunk) is str and chunk for chunk in chunks) + ): + raise ValueError("transport planner returned invalid chunks") + return list(chunks) + + +def _persist_target_text_receipts( + receipts: Any, + attempts: dict, + requested_target: dict[str, str], + components: Optional[set[str]] = None, + expected_actual_target: Optional[dict[str, str]] = None, +) -> bool: + """Persist exact acknowledgements and prove the selected planned set. + + Matching partial acknowledgements are retained even when the final result + is false. ``components=None`` requires every preregistered component; + callers passing a set require only those component kinds. + """ + from gateway.platforms.base import TransportReceipt, TransportTarget + + if type(receipts) is not tuple: + return False + if type(attempts) is not dict or type(requested_target) is not dict: + return False + if components is not None and type(components) is not set: + return False + if expected_actual_target is not None and type(expected_actual_target) is not dict: + return False + if not all(type(receipt) is TransportReceipt for receipt in receipts): + return False + if not attempts: + return bool(receipts) + expected = { + key for key in attempts + if key[:3] == ( + requested_target["platform"], requested_target["chat_id"], + requested_target["thread_id"], + ) and (components is None or key[3] in components) + } + observed = set() + persisted_all = True + expected_actual = expected_actual_target or requested_target + try: + planned_target = ( + expected_actual["platform"], + expected_actual["chat_id"], + expected_actual["thread_id"], + ) + except (KeyError, TypeError): + return False + if not all(type(value) is str for value in planned_target): + return False + for receipt in receipts: + try: + requested = receipt.requested_target + key = ( + requested.platform, requested.chat_id, + requested.thread_id or "", receipt.component, receipt.ordinal, + ) + attempt_id = attempts.get(key) + persisted = bool(attempt_id) and record_transport_receipt(attempt_id, receipt) + except Exception: + persisted = False + key = None + actual = receipt.actual_target + actual_target = ( + (actual.platform, actual.chat_id, actual.thread_id or "") + if type(actual) is TransportTarget + else None + ) + if ( + persisted + and key is not None + and receipt.outcome == "delivered" + and actual_target == planned_target + ): + observed.add(key) + else: + persisted_all = False + return bool(expected) and persisted_all and observed == expected + + +def _receipt_delivery_outcome(execution_id: str) -> Optional[str]: + """Project a transport outcome only when this execution has a receipt plan.""" + try: + counts = receipt_summary(execution_id) + except Exception: + return None + if counts.get("unknown", 0) > 0: + return "unknown" + if counts.get("failed", 0) > 0: + return "failed" + if counts.get("delivered", 0) > 0 and counts.get("targets_delivered", 0) > 0: + return "delivered" + return None + + def _cron_delivery_notify_enabled(cfg: Optional[dict]) -> bool: """Resolve ``cron.delivery.notify`` (config.yaml). Default True. @@ -3180,7 +3425,15 @@ def _record_delivery_verification(job: dict, unverified_targets: list) -> None: def _deliver_result( - job: dict, content: str, adapters=None, loop=None, *, for_failure: bool = False + job: dict, + content: str, + adapters=None, + loop=None, + *, + execution_id: Optional[str] = None, + fire_identity: Optional[str] = None, + for_failure: bool = False, + ) -> Optional[str]: """ Deliver job output to the configured target(s) (origin chat, specific platform, etc.). @@ -3195,7 +3448,12 @@ def _deliver_result( Returns None on success, or an error string on failure. """ - targets = _resolve_delivery_targets(job, for_failure=for_failure) + if type(job) is not dict or type(content) is not str: + return "delivery input is invalid; no delivery was sent" + try: + targets = _resolve_delivery_targets(job, for_failure=for_failure) + except (TypeError, ValueError): + return "delivery target is invalid; no delivery was sent" if not targets: deliver_value = _normalize_deliver_value( _delivery_lane_value(job, for_failure=for_failure) @@ -3281,7 +3539,12 @@ def _deliver_result( delivery_content = content # Extract MEDIA: tags so attachments are forwarded as files, not raw text - from gateway.platforms.base import BasePlatformAdapter + from gateway.platforms.base import ( + BasePlatformAdapter, + SendResult, + TransportReceipt, + TransportTarget, + ) # Bridge gateway media-policy config (strict / allow_dirs / trust_recent) # into the env vars the path validator reads. Gateway startup does this @@ -3336,11 +3599,91 @@ def _deliver_result( return msg delivery_errors = [] + # Direct isolated callers may opt out of ledger persistence. Scheduler, + # provider, and manual routes pass the exact durable identity explicitly; + # never rely on mutating their job snapshot. + receipt_attempts = {} + if execution_id is None: + execution_id = job.get("execution_id") + if fire_identity is None: + fire_identity = job.get("fire_identity") + if fire_identity is None: + fire_identity = execution_id + receipt_planning_adapters = ( + adapters + if adapters is not None + and loop is not None + and getattr(loop, "is_running", lambda: False)() + else None + ) + if execution_id is not None: + if ( + type(execution_id) is not str + or not execution_id + or type(fire_identity) is not str + or not fire_identity + ): + return "delivery receipt identity is invalid; no delivery was sent" + receipt_plan = [] + for target in targets: + target_identity = dict(target) + target_identity["thread_id"] = target_identity["thread_id"] or "" + if target_identity["platform"] == BOT_CHAT_PLATFORM: + receipt_plan.append({ + "target": target_identity, + "component": "text", + "ordinal": 0, + "content": _bot_chat_query_message(job, content), + }) + continue + if cleaned_delivery_content.strip(): + try: + planned_chunks = _receipt_text_chunks_for_target( + receipt_planning_adapters, target["platform"], + cleaned_delivery_content.strip(), + media_files=media_files, + ) + except (TypeError, ValueError): + return "delivery receipt planner is invalid; no delivery was sent" + for ordinal, chunk in enumerate(planned_chunks): + receipt_plan.append({ + "target": target_identity, "component": "text", "ordinal": ordinal, + "content": chunk, + }) + for ordinal, (media_path, _is_voice) in enumerate(media_files): + if type(media_path) is not str: + return "delivery media identity is invalid; no delivery was sent" + receipt_plan.append({ + "target": target_identity, "component": "media", "ordinal": ordinal, + "content": media_path, + }) + if receipt_plan: + try: + attempts = preregister_receipt_plan( + execution_id, + fire_identity=fire_identity, + components=receipt_plan, + ) + except Exception: + # The DB exception can include filesystem/provider details; it + # is not a safe delivery/operator payload. + logger.warning("Job '%s': receipt-plan preregistration failed", job["id"]) + return "delivery receipt plan could not be persisted; no delivery was sent" + for attempt in attempts: + receipt_attempts[( + attempt["platform"], attempt["chat_id"], attempt["thread_id"], + attempt["component"], attempt["ordinal"], + )] = attempt["id"] for target in targets: platform_name = target["platform"] chat_id = target["chat_id"] thread_id = target.get("thread_id") + receipt_requested_target = { + "platform": platform_name, + "chat_id": chat_id, + "thread_id": thread_id or "", + } # bot-chat targets don't ride a gateway adapter: the output becomes a # real inbound turn in the target profile's canonical Bot Chat via the @@ -3348,8 +3691,56 @@ def _deliver_result( # bot runs a turn and can respond — handled before the Platform enum # below, which knows nothing about this pseudo-platform. if platform_name == BOT_CHAT_PLATFORM: - bot_chat_error = _deliver_to_bot_chat(job, content, chat_id) - if bot_chat_error: + bot_chat_profile = "" if chat_id == BOT_CHAT_SELF_TARGET else chat_id + bot_chat_error = _deliver_to_bot_chat(job, content, bot_chat_profile) + if receipt_attempts: + requested = TransportTarget( + BOT_CHAT_PLATFORM, + chat_id, + thread_id, + ) + if bot_chat_error == "bot-chat delivery failed": + receipt = TransportReceipt( + outcome="failed", + requested_target=requested, + failure_kind="pre_dispatch", + component="text", + ordinal=0, + ) + else: + # Child completion has no provider/session acknowledgement; + # any post-spawn result remains ambiguous. + receipt = TransportReceipt( + outcome="unknown", + requested_target=requested, + component="text", + ordinal=0, + ) + if receipt.outcome == "unknown": + attempt_id = receipt_attempts.get(( + BOT_CHAT_PLATFORM, chat_id, + thread_id or "", "text", 0, + )) + persisted = bool(attempt_id) and observe_transport_unknown( + attempt_id, receipt, + ) + else: + persisted = _persist_target_text_receipts( + (receipt,), receipt_attempts, receipt_requested_target, + components={"text"}, + ) + if not persisted: + delivery_errors.append( + "bot-chat delivery receipt could not be persisted; " + "delivery is unknown" + ) + elif receipt.outcome == "unknown": + delivery_errors.append( + "bot-chat delivery confirmation unavailable" + ) + else: + delivery_errors.append("bot-chat delivery failed") + elif bot_chat_error: delivery_errors.append(bot_chat_error) continue @@ -3489,6 +3880,7 @@ def _deliver_result( ) delivered = False target_errors = [] + ambiguous_live_timeout = False # Continuable cron surface (D1/D2/D6): resolve the delivery surface for # this platform generically from its config ``extra``. Default "thread" @@ -3690,6 +4082,12 @@ def _deliver_result( media_metadata = dict(media_metadata or {}) media_metadata.setdefault("scope_id", str(origin["scope_id"])) + # Provider routing may create, flatten, or fall back from a thread + # after the global receipt plan was durably registered. Preserve + # the logical requested identity separately; adapters record the + # routed destination as actual_target. + route_metadata["_transport_receipt_requested_target"] = receipt_requested_target + try: # Send cleaned text (MEDIA tags stripped) — not the raw content. # Route through the gateway's DeliveryRouter so the live send @@ -3702,6 +4100,8 @@ def _deliver_result( adapter_ok = True timed_out = False delivered_message_id = None + send_result = None + send_receipts = () if not text_to_send and not media_files: # Nothing to hand the adapter at all. This used to fall # straight through to the `if adapter_ok:` branch below and @@ -3747,52 +4147,40 @@ def _deliver_result( try: send_result = future.result(timeout=60) except TimeoutError: - # #38922: a slow confirmation does NOT necessarily - # mean the send failed — but we must distinguish two - # cases via future.cancel()'s return value: - # - # cancel() == False -> the coroutine was already - # running on the gateway loop when the timeout - # fired; the request is in flight on the wire and - # cannot be un-sent. Re-sending via standalone - # would be a guaranteed DUPLICATE, so treat it as - # delivered (assume-delivered). - # - # cancel() == True -> the scheduled callback never - # started executing (loop wedged/backlogged for - # the full 60s), so nothing was sent. We MUST - # fall through to the standalone path or the - # message is silently dropped (worse than a - # duplicate). - cancelled = future.cancel() - if cancelled: - msg = ( - f"live adapter send to {platform_name}:{chat_id} " - "timed out before the coroutine was dispatched" - ) - logger.warning( - "Job '%s': %s, falling back to standalone", - job["id"], msg, - ) - target_errors.append(msg) - adapter_ok = False # fall through to standalone path - timeout_handled = True - else: - timed_out = True - timeout_handled = True - logger.warning( - "Job '%s': live adapter send to %s:%s timed out " - "after 60s; already dispatched (in flight), " - "assuming delivered (skipping standalone fallback " - "to avoid duplicate)", - job["id"], platform_name, chat_id, - ) + # Cancellation only describes the local Future's + # state. It is neither an acknowledgement from the + # provider nor proof that no request crossed the + # wire. Conservatively classify either result as + # unknown and prohibit same-identity fallback. + future.cancel() + timed_out = True + timeout_handled = True + ambiguous_live_timeout = True + adapter_ok = False + msg = ( + f"live adapter confirmation timed out for " + f"{platform_name}:{chat_id}; delivery is unknown" + ) + target_errors.append(msg) + logger.warning("Job '%s': %s", job["id"], msg) except Exception as ex: - # A real send error (not a slow confirmation) — fall - # through to the standalone path so the message is - # still delivered. target_errors.append(f"live adapter send failed: {ex}") - raise + # Exceptions do not prove a provider request was not + # dispatched. Keep this target unknown and prohibit + # same-identity fallback. + ambiguous_live_timeout = True + partial_result = getattr(ex, "send_result", None) + if partial_result is None: + raise + # DeliveryRouter preserves a failed SendResult when + # it contains provider acknowledgements for earlier + # chunks. Skip success normalization but retain those + # receipts below. + send_result = partial_result + if type(partial_result) is SendResult: + send_receipts = partial_result.receipts + adapter_ok = False + timeout_handled = True if timeout_handled: # The timeout branch above already decided the @@ -3808,16 +4196,35 @@ def _deliver_result( # {"success": True, "delivered": False, ...}. # Normalize both shapes so a getattr default doesn't # misread a dict, and so a None / success-less object - # is NOT counted as delivered (#47056). The - # confirmation itself handles both shapes: a truthy - # `success` with `delivered: False` is a drop, not a - # delivery (#77763). - if isinstance(send_result, dict): - send_raw_response = send_result.get("raw_response") - delivered_message_id = send_result.get("message_id") + # Normalize only inert/known result containers. A + # filtered dict with delivered=False is not a delivery; + # a successful send without provider evidence remains + # explicitly UNVERIFIED. + send_receipts = () + legacy_fields = None + send_raw_response = None + delivered_message_id = None + if type(send_result) is dict: + raw_response_value = send_result.get("raw_response") + send_raw_response = ( + raw_response_value + if type(raw_response_value) is dict else None + ) + message_id_value = send_result.get("message_id") + delivered_message_id = ( + message_id_value + if type(message_id_value) in {str, int} else None + ) + elif type(send_result) is SendResult: + send_raw_response = send_result.raw_response + delivered_message_id = send_result.message_id + send_receipts = send_result.receipts else: - send_raw_response = getattr(send_result, "raw_response", None) - delivered_message_id = getattr(send_result, "message_id", None) + legacy_fields = _inert_legacy_send_result_fields(send_result) + if legacy_fields is not None: + send_raw_response = legacy_fields["raw_response"] + delivered_message_id = legacy_fields["message_id"] + _evidence_gap: list = [] send_success = _confirm_adapter_delivery( send_result, job["id"], _evidence_gap, @@ -3826,18 +4233,27 @@ def _deliver_result( unverified_targets.append(f"{platform_name}:{chat_id}") if not send_success: - if isinstance(send_result, dict): - # A filtered drop carries no "error" — name - # the filter instead of reporting "unknown". + if type(send_result) is dict: + error_value = send_result.get("error") + filtered_value = send_result.get("filtered") err = ( - send_result.get("error") - or send_result.get("filtered") - or "unknown" + error_value + if type(error_value) is str + else filtered_value + if type(filtered_value) is str + else "unknown" + ) shape = "dict" + elif type(send_result) is SendResult: + err = send_result.error + shape = "SendResult" + elif legacy_fields is not None: + err = legacy_fields["error"] or "unknown" + shape = "legacy" elif send_result is not None: - err = getattr(send_result, "error", None) - shape = type(send_result).__name__ + err = "invalid adapter result" + shape = "invalid" else: err = "no response from adapter" shape = "None" @@ -3853,7 +4269,28 @@ def _deliver_result( job["id"], msg, ) target_errors.append(msg) - adapter_ok = False # fall through to standalone path + # A negative legacy result does not prove the + # request never crossed the provider boundary. + # Preserve any earlier typed chunk receipts and + # never blind-resend this execution identity. + ambiguous_live_timeout = True + adapter_ok = False + elif not send_receipts and ( + receipt_attempts or type(send_result) is SendResult + ): + # ``success``/``message_id`` are legacy operation + # fields, not provider acknowledgement evidence. + # A same-identity retry could duplicate a write + # which completed before an old adapter returned. + ambiguous_live_timeout = True + adapter_ok = False + msg = ( + f"live adapter send to {platform_name}:{chat_id} " + "returned legacy success without typed receipt; " + "delivery is unknown" + ) + target_errors.append(msg) + logger.warning("Job '%s': %s", job["id"], msg) elif ( send_raw_response and thread_id @@ -3867,6 +4304,32 @@ def _deliver_result( logger.warning("Job '%s': %s", job["id"], msg) delivery_errors.append(msg) + # A typed acknowledgement must be committed before any follow-up + # send, fallback, mirror, or seed. A database error leaves its + # preregistered attempt unknown and makes retry unsafe. + if text_to_send and receipt_attempts and send_result is not None: + persisted_all = _persist_target_text_receipts( + send_receipts, + receipt_attempts, + receipt_requested_target, + components={"text"}, + expected_actual_target={ + "platform": platform_name, + "chat_id": chat_id, + "thread_id": ( + str(route_metadata["direct_messages_topic_id"]) + if route_metadata.get("direct_messages_topic_id") is not None + else route_thread_id or "" + ), + }, + ) + if not persisted_all: + ambiguous_live_timeout = True + adapter_ok = False + target_errors.append( + f"live adapter acknowledgement for {platform_name}:{chat_id} could not be persisted; delivery is unknown" + ) + # Send extracted media files as native attachments via the live # adapter, using the same DM-topic-aware routing as the text send # (#22773 — media previously used a bare thread_id and landed in @@ -3876,6 +4339,8 @@ def _deliver_result( # payload is already assumed delivered (#38922). Record the # skipped attachments so the drop is visible rather than silently # lost. + _media_receipts = [] + _media_errors = [] if adapter_ok and not timed_out and media_files: routed_media_metadata = dict(media_metadata or {}) if transport is not None and transport.is_relay: @@ -3886,6 +4351,9 @@ def _deliver_result( routed_media_metadata["user_id"] = logical_home.user_id if logical_home.scope_id: routed_media_metadata["scope_id"] = logical_home.scope_id + routed_media_metadata["_transport_receipt_requested_target"] = ( + receipt_requested_target + ) _media_errors = _send_media_via_adapter( runtime_adapter, chat_id, @@ -3894,6 +4362,7 @@ def _deliver_result( loop, job, platform=platform, + receipts_out=_media_receipts, ) # Surface per-file failures into the run status (parity # with the standalone lane): text delivered but an @@ -3909,6 +4378,39 @@ def _deliver_result( logger.warning("Job '%s': %s", job["id"], msg) delivery_errors.append(msg) + media_receipts_persisted = bool(media_files) and _persist_target_text_receipts( + tuple(_media_receipts) if not timed_out else (), + receipt_attempts, + receipt_requested_target, + components={"media"}, + expected_actual_target={ + "platform": platform_name, + "chat_id": chat_id, + "thread_id": ( + str(route_metadata["direct_messages_topic_id"]) + if route_metadata.get("direct_messages_topic_id") is not None + else route_thread_id or "" + ), + }, + ) + if _media_errors: + # A failed SendResult does not prove that the provider did + # not accept the media. Preserve any receipts above, but + # never retry the whole target through the standalone lane: + # that could duplicate text or attachments after an + # ambiguous live-adapter write. + ambiguous_live_timeout = True + adapter_ok = False + if media_files and not media_receipts_persisted: + # Preserve any partial typed acknowledgements, but keep the + # target unknown unless every planned media component was + # confirmed and persisted. + ambiguous_live_timeout = True + adapter_ok = False + target_errors.append( + f"media acknowledgement for {platform_name}:{chat_id} is unavailable; delivery is partial" + ) + if adapter_ok: # Log WHERE it went, not just that it went: a ghost delivery # that landed in the wrong lane (General topic instead of the @@ -4010,6 +4512,12 @@ def _deliver_result( job["id"], err_msg, ) + if ambiguous_live_timeout: + # No standalone retry, mirror, or seed after an ambiguous live + # send: any of them could duplicate an unconfirmed provider write. + delivery_errors.extend(target_errors) + continue + if not delivered: if transport is not None and transport.is_relay: # Relay owns the logical destination and its connector owns the @@ -4050,7 +4558,11 @@ def _deliver_result( delivery_errors.extend(target_errors) continue # Standalone path: run the async send in a fresh event loop (safe from any thread) - coro = _send_to_platform(platform, pconfig, chat_id, cleaned_delivery_content, thread_id=thread_id, media_files=media_files) + coro = _send_to_platform( + platform, pconfig, chat_id, cleaned_delivery_content, + thread_id=thread_id, media_files=media_files, + receipt_bound=bool(receipt_attempts), + ) try: result = asyncio.run(coro) except RuntimeError as run_err: @@ -4079,6 +4591,13 @@ def _deliver_result( try: pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) try: + def _run_standalone_send(): + return asyncio.run(_send_to_platform( + platform, pconfig, chat_id, cleaned_delivery_content, + thread_id=thread_id, media_files=media_files, + receipt_bound=bool(receipt_attempts), + )) + # The fallback worker is a fresh thread: it does NOT # inherit the multiplexed profile ContextVars (home # override + secret scope). Run inside a copy of the @@ -4089,8 +4608,7 @@ def _deliver_result( _fallback_context = contextvars.copy_context() future = pool.submit( _fallback_context.run, - asyncio.run, - _send_to_platform(platform, pconfig, chat_id, cleaned_delivery_content, thread_id=thread_id, media_files=media_files), + _run_standalone_send, ) result = future.result(timeout=30) finally: @@ -4116,6 +4634,27 @@ def _deliver_result( delivery_errors.extend(target_errors) continue + if receipt_attempts: + standalone_receipts = ( + result.get("receipts", ()) if isinstance(result, dict) else () + ) + if not _persist_target_text_receipts( + standalone_receipts, receipt_attempts, receipt_requested_target, + ): + if media_files: + msg = ( + f"media acknowledgement for {platform_name}:{chat_id} " + "is unavailable; delivery is partial" + ) + else: + msg = ( + f"standalone send to {platform_name}:{chat_id} returned " + "without a complete typed receipt; delivery is unknown" + ) + target_errors.append(msg) + delivery_errors.extend(target_errors) + continue + if result and result.get("error"): # Include target context (platform/chat) so a bare error string # like "Discord send failed: TimeoutError: " is attributable. @@ -7498,6 +8037,17 @@ def run_one_job( _running_fire_owners.pop(job["id"], None) +def _claimed_fire_identity(job: dict) -> Optional[str]: + """Resolve the immutable identity of an already-acquired fire claim.""" + claim = job.get("fire_claim") + if not isinstance(claim, dict): + return None + fire_at = claim.get("fire_at") + if not isinstance(fire_at, str) or not fire_at: + raise ValueError("acquired fire claim has no immutable timestamp") + return scheduled_fire_identity(job["id"], fire_at) + + def _run_one_job_body( job: dict, *, @@ -7538,9 +8088,28 @@ def _fire_claim_ownership_lost() -> bool: fire_claim_lost.set() return True + expected_fire_identity = _claimed_fire_identity(job) execution_id = job.get("execution_id") if not execution_id: - execution_id = create_execution(job["id"], source="direct")["id"] + create_kwargs = {"source": "direct"} + if expected_fire_identity is not None: + create_kwargs["fire_identity"] = expected_fire_identity + execution = create_execution(job["id"], **create_kwargs) + execution_id = execution["id"] + job["execution_id"] = execution_id + job["fire_identity"] = ( + execution.get("fire_identity") or expected_fire_identity or execution_id + ) + elif expected_fire_identity is not None: + # The job snapshot is not authoritative for durable binding. Always ask + # the ledger to verify or atomically bind before running; the operation + # is idempotent only when the persisted identity already matches. + bound_execution = bind_execution_fire_identity( + execution_id, expected_fire_identity, + ) + job["fire_identity"] = ( + bound_execution.get("fire_identity") or expected_fire_identity + ) delivery_attempted = False delivery_error = None # Durable failure-incident bookkeeping for this run (see cron.incidents): @@ -7826,6 +8395,8 @@ def _fire_claim_ownership_lost() -> bool: deliver_content, adapters=adapters, loop=loop, + execution_id=str(execution_id), + fire_identity=str(job.get("fire_identity") or execution_id), # Failure summaries (and drift/blocked-config alerts # composed into deliver_content on the failure path) # honor the job's failure_deliver override (NS-788). @@ -7920,8 +8491,15 @@ def _fire_claim_ownership_lost() -> bool: normalized_deliver = _normalize_deliver_value( _delivery_lane_value(job, for_failure=not success) ) - if delivery_error: + receipt_outcome = ( + _receipt_delivery_outcome(str(execution_id)) if should_deliver else None + ) + if receipt_outcome == "unknown": + delivery_outcome = "unknown" + elif delivery_error: delivery_outcome = "failed" + elif receipt_outcome is not None: + delivery_outcome = receipt_outcome elif should_deliver and unresolved_origin: delivery_outcome = "not_configured" elif should_deliver and normalized_deliver != "local": @@ -7979,9 +8557,8 @@ def _fire_claim_ownership_lost() -> bool: _delivery_lane_value(job, for_failure=True) ) unresolved_origin = False - # Durable failure incident: same ack gate as the normal failure - # delivery above — an acked signature stays silent on this path - # too, so the retry-path alert cannot re-ping after acknowledgment. + # Durable failure incident: preserve upstream ack suppression while + # keeping the receipt-bound delivery contract for this PR. incident_acked, failure_incident_id = _upsert_incident_for_failure( job, _err_text ) @@ -8002,6 +8579,8 @@ def _fire_claim_ownership_lost() -> bool: + _failure_streak_nudge(job), adapters=adapters, loop=loop, + execution_id=str(execution_id), + fire_identity=str(job.get("fire_identity") or execution_id), for_failure=True, ) except Exception as delivery_exc: @@ -8013,12 +8592,15 @@ def _fire_claim_ownership_lost() -> bool: unresolved_origin = not _resolve_delivery_targets( job, for_failure=True ) - if delivery_error: + receipt_outcome = _receipt_delivery_outcome(str(execution_id)) + if receipt_outcome == "unknown": + delivery_outcome = "unknown" + elif receipt_outcome == "failed" or delivery_error: delivery_outcome = "failed" elif unresolved_origin: delivery_outcome = "not_configured" elif normalized_deliver != "local": - delivery_outcome = "delivered" + delivery_outcome = receipt_outcome or "delivered" if delivery_outcome in ("delivered", "not_configured"): _mark_incident_alerted(failure_incident_id) try: @@ -8412,7 +8994,7 @@ def user_message(self) -> str: def to_dict(self) -> dict: """Return the public partial-failure contract without provider details.""" return { - "error": str(self), + "error": "scheduler_registration_failed", "job_id": self.job["id"], "job_saved": True, "scheduler_registered": False, @@ -8782,6 +9364,29 @@ def _process_job(job: dict) -> bool: # compatible; real callers using return_job=True never take it. claimed_job = dict(claimed) if isinstance(claimed, dict) else dict(job) claimed_job["execution_id"] = job["execution_id"] + if isinstance(claimed, dict): + try: + expected_fire_identity = _claimed_fire_identity(claimed_job) + bound_execution = bind_execution_fire_identity( + job["execution_id"], expected_fire_identity, + ) + claimed_job["fire_identity"] = ( + bound_execution.get("fire_identity") or expected_fire_identity + ) + except BaseException as exc: + finish_execution( + job["execution_id"], + success=False, + error=( + "Fire identity binding failed before dispatch: " + f"{type(exc).__name__}: {exc}" + ), + ) + raise + else: + claimed_job["fire_identity"] = ( + job.get("fire_identity") or job["execution_id"] + ) return run_one_job( claimed_job, adapters=adapters, @@ -8851,7 +9456,10 @@ def _clear_run_claim_best_effort() -> None: # abandoned records as unknown; it never automatically retries them. try: execution = create_execution(job_id, source="builtin") - dispatched_job = dict(job, execution_id=execution["id"]) + dispatched_job = dict( + job, + execution_id=execution["id"], + ) _ctx = contextvars.copy_context() except Exception as execution_err: # Init/creation failure between the claim and the submit — diff --git a/cron/scheduler_provider.py b/cron/scheduler_provider.py index 563cba345b2d..816a0fc7072f 100644 --- a/cron/scheduler_provider.py +++ b/cron/scheduler_provider.py @@ -205,10 +205,29 @@ def claim_fire(self, job_id: str, *, force: bool = False) -> dict | None: external scheduler, then pass the exact owner-bearing snapshot to ``fire_claimed`` in tracked background work. """ - from cron.executions import create_execution, finish_execution - from cron.jobs import claim_job_for_fire + from cron.executions import ( + bind_execution_fire_identity, + create_execution, + finish_execution, + scheduled_fire_identity, + ) + from cron.jobs import claim_job_for_fire, get_job + + preclaim_job = get_job(job_id) + recovery_fire_identity = None + if not force and isinstance(preclaim_job, dict): + existing_claim = preclaim_job.get("fire_claim") + if isinstance(existing_claim, dict): + if not existing_claim.get("fire_at"): + raise ValueError("existing fire claim has no immutable timestamp") + recovery_fire_identity = scheduled_fire_identity( + job_id, existing_claim["fire_at"], + ) - execution = create_execution(job_id, source=self.name) + create_kwargs = {"source": self.name} + if recovery_fire_identity is not None: + create_kwargs["fire_identity"] = recovery_fire_identity + execution = create_execution(job_id, **create_kwargs) claim_kwargs = {"return_job": True} if force: claim_kwargs["force"] = True @@ -226,9 +245,35 @@ def claim_fire(self, job_id: str, *, force: bool = False) -> dict | None: execution["id"], success=False, error="Fire claim was not acquired", + error_kind="claim_lost", ) return None + bound_execution = execution + if recovery_fire_identity is None: + try: + acquired_claim = claimed_job.get("fire_claim") + if ( + not isinstance(acquired_claim, dict) + or not acquired_claim.get("fire_at") + ): + raise ValueError("acquired fire claim has no immutable timestamp") + acquired_fire_identity = scheduled_fire_identity( + job_id, acquired_claim["fire_at"], + ) + bound_execution = bind_execution_fire_identity( + execution["id"], acquired_fire_identity, + ) + except BaseException as exc: + finish_execution( + execution["id"], + success=False, + error=f"Fire identity binding failed before dispatch: {type(exc).__name__}: {exc}", + ) + raise claimed_job["execution_id"] = execution["id"] + claimed_job["fire_identity"] = ( + bound_execution.get("fire_identity") or execution["id"] + ) return claimed_job def fire_claimed( diff --git a/gateway/delivery.py b/gateway/delivery.py index f63c769aeb2c..01b7d12ce84a 100644 --- a/gateway/delivery.py +++ b/gateway/delivery.py @@ -89,6 +89,14 @@ async def send( return await self.adapter.send(chat_id, content, metadata=metadata) +class DeliverySendError(RuntimeError): + """Delivery failure that preserves content-free typed receipt evidence.""" + + def __init__(self, message: str, send_result: Any): + super().__init__(message) + self.send_result = send_result + + def resolve_delivery_transport( platform: Platform, config: GatewayConfig, @@ -649,7 +657,10 @@ async def _deliver_to_platform( metadata=send_metadata or None, ) if _send_result_failed(result): - raise RuntimeError(_send_result_error(result) or f"{target.platform.value} delivery failed") + raise DeliverySendError( + _send_result_error(result) or f"{target.platform.value} delivery failed", + result, + ) return result diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index d2ac5996e601..a1bfbecf6d21 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -18,6 +18,7 @@ import tempfile import threading import time +import unicodedata import uuid import weakref from abc import ABC, abstractmethod @@ -646,7 +647,7 @@ def is_host_excluded_by_no_proxy(hostname: str, no_proxy_value: str | None = Non import dataclasses from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Any, Callable, Awaitable, Tuple, Union from enum import Enum @@ -2679,13 +2680,169 @@ def coerce_plaintext_gateway_command(event: "MessageEvent") -> None: return +TRANSPORT_RECEIPT_OUTCOMES = frozenset({"delivered", "unknown", "failed"}) +TRANSPORT_RECEIPT_FAILURE_KINDS = frozenset( + { + "bad_format", + "forbidden", + "invalid_target", + "not_configured", + "not_found", + "pre_dispatch", + "provider_rejected", + } +) + + +def normalize_transport_provider_message_id(value: Any) -> Optional[str]: + """Return an inert bounded provider acknowledgement id, or ``None``.""" + if type(value) is str: + candidate = value + elif type(value) is int: + candidate = str(value) + else: + value_type = type(value) + try: + value_mro = type.__getattribute__(value_type, "__mro__") + except (AttributeError, TypeError): + return None + if type(value_mro) is not tuple or not any( + base is str for base in value_mro + ): + return None + candidate = str.__str__(value) + if ( + not candidate + or len(candidate) > 1024 + or candidate != unicodedata.normalize("NFC", candidate) + or any( + char.isspace() or unicodedata.category(char).startswith("C") + for char in candidate + ) + ): + return None + return candidate + + +@dataclass(frozen=True) +class TransportTarget: + """A bounded, content-free identity for one provider delivery target.""" + + platform: str + chat_id: str + thread_id: Optional[str] = None + + def __post_init__(self) -> None: + for name, value, limit in ( + ("platform", self.platform, 64), + ("chat_id", self.chat_id, 512), + ("thread_id", self.thread_id, 512), + ): + if value is None and name == "thread_id": + continue + if ( + type(value) is not str + or not value + or len(value) > limit + or value != unicodedata.normalize("NFC", value) + or any(char.isspace() or unicodedata.category(char).startswith("C") for char in value) + ): + raise ValueError(f"{name} must be a non-empty string of at most {limit} characters") + + +@dataclass(frozen=True) +class TransportReceipt: + """Provider acknowledgement without payloads, raw errors, or credentials. + + ``SendResult.success`` remains a legacy transport-operation signal. A + receipt is deliberately separate: absent receipt means unknown, never an + implied delivery acknowledgement. + """ + + outcome: str + requested_target: TransportTarget + actual_target: Optional[TransportTarget] = None + provider_message_id: Optional[str] = None + observed_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + failure_kind: Optional[str] = None + component: str = "text" + ordinal: int = 0 + + def __post_init__(self) -> None: + if type(self.requested_target) is not TransportTarget: + raise TypeError("requested_target must be a TransportTarget") + if self.actual_target is not None and type(self.actual_target) is not TransportTarget: + raise TypeError("actual_target must be a TransportTarget") + if type(self.outcome) is not str or self.outcome not in TRANSPORT_RECEIPT_OUTCOMES: + raise ValueError("outcome must be delivered, unknown, or failed") + if self.provider_message_id is not None: + if ( + type(self.provider_message_id) is not str + or not self.provider_message_id + or len(self.provider_message_id) > 1024 + or self.provider_message_id != unicodedata.normalize("NFC", self.provider_message_id) + or any( + char.isspace() or unicodedata.category(char).startswith("C") + for char in self.provider_message_id + ) + ): + raise ValueError("provider_message_id must be a non-empty string of at most 1024 characters") + if ( + type(self.component) is not str + or not self.component + or len(self.component) > 64 + or any(char.isspace() or unicodedata.category(char).startswith("C") for char in self.component) + or type(self.ordinal) is not int + or self.ordinal < 0 + ): + raise ValueError("receipt component must be a bounded name with non-negative ordinal") + if ( + type(self.observed_at) is not datetime + or self.observed_at.tzinfo is None + or type(self.observed_at.tzinfo) is not timezone + ): + raise ValueError("observed_at must be timezone-aware") + observed_utc = self.observed_at.astimezone(timezone.utc) + if ( + observed_utc < datetime(1970, 1, 1, tzinfo=timezone.utc) + or observed_utc > datetime.now(timezone.utc) + timedelta(minutes=5) + ): + raise ValueError("observed_at must be a bounded observation time") + if self.failure_kind is not None and type(self.failure_kind) is not str: + raise ValueError("failure_kind must be a bounded category") + if self.outcome == "delivered": + if not self.provider_message_id: + raise ValueError("delivered receipt requires provider_message_id") + if self.actual_target is None: + raise ValueError("delivered receipt requires actual_target") + if self.failure_kind is not None: + raise ValueError("delivered receipt cannot include failure_kind") + elif self.outcome == "failed": + if self.failure_kind not in TRANSPORT_RECEIPT_FAILURE_KINDS: + raise ValueError("failed receipt requires a bounded failure_kind") + if self.provider_message_id is not None: + raise ValueError("failed receipt cannot include provider evidence") + if self.actual_target is not None: + raise ValueError("failed receipt cannot include actual_target") + elif self.failure_kind is not None: + raise ValueError("unknown receipt cannot include failure_kind") + elif self.provider_message_id is not None or self.actual_target is not None: + raise ValueError("unknown receipt cannot include provider evidence") + + @dataclass class SendResult: - """Result of sending a message.""" + """Result of sending a message. + + ``receipt`` is additive and optional for adapter compatibility. Consumers + requiring a provider acknowledgement must treat ``None`` as unknown. + """ success: bool message_id: Optional[str] = None error: Optional[str] = None raw_response: Any = None + receipt: Optional[TransportReceipt] = None + receipts: Tuple[TransportReceipt, ...] = () # Adapter-specific metadata. Cross-layer contracts that affect delivery # semantics must be documented at the producer and consumer sites. Current # known contract: Telegram edit overflow partials set @@ -2712,6 +2869,60 @@ class SendResult: # :func:`classify_send_error`. error_kind: Optional[str] = None + def __post_init__(self) -> None: + """Normalize the additive receipt tuple without upgrading legacy ids.""" + if type(self.receipts) is not tuple: + raise ValueError("receipts must be an immutable tuple of TransportReceipt values") + + def rebuild(item: Any) -> TransportReceipt: + if type(item) is not TransportReceipt: + raise ValueError( + "receipts must be an immutable tuple of TransportReceipt values" + ) + requested = item.requested_target + actual = item.actual_target + if type(requested) is not TransportTarget: + raise ValueError("receipt requested_target must be a TransportTarget") + if actual is not None and type(actual) is not TransportTarget: + raise ValueError("receipt actual_target must be a TransportTarget") + requested = TransportTarget( + requested.platform, requested.chat_id, requested.thread_id, + ) + if actual is not None: + actual = TransportTarget( + actual.platform, actual.chat_id, actual.thread_id, + ) + return TransportReceipt( + outcome=item.outcome, + requested_target=requested, + actual_target=actual, + provider_message_id=item.provider_message_id, + observed_at=item.observed_at, + failure_kind=item.failure_kind, + component=item.component, + ordinal=item.ordinal, + ) + + receipts = tuple(rebuild(item) for item in self.receipts) + receipt = rebuild(self.receipt) if self.receipt is not None else None + if receipt is not None: + if receipts and receipt not in receipts: + raise ValueError("receipt must be included in receipts") + if not receipts: + receipts = (receipt,) + elif receipts: + receipt = receipts[0] + self.receipt = receipt + self.receipts = receipts + expected = sorted( + receipts, + key=lambda item: (item.component, item.ordinal), + ) + if list(receipts) != expected: + raise ValueError("receipts must be ordered by component and ordinal") + if len({(item.component, item.ordinal) for item in receipts}) != len(receipts): + raise ValueError("receipts must not duplicate a component ordinal") + # Machine-readable send-failure categories. Kept platform-neutral so every # adapter can populate ``SendResult.error_kind`` from the same vocabulary and diff --git a/hermes_cli/cli_commands_mixin.py b/hermes_cli/cli_commands_mixin.py index b78ff6157c16..6000b1a401df 100644 --- a/hermes_cli/cli_commands_mixin.py +++ b/hermes_cli/cli_commands_mixin.py @@ -1990,7 +1990,11 @@ def _parse_flags(tokens): # delivery_failed: the agent ran fine but the output never # reached the target — name the delivery reason, which # lives in last_delivery_error (last_error is None). - if status == "delivery_failed" and job.get("last_delivery_error"): + if ( + status == "delivery_failed" + and job.get("last_delivery_error") + and job["last_delivery_error"] != "delivery_failed" + ): status = f"delivery_failed: {job['last_delivery_error']}" print(f" Last run: {job['last_run_at']} ({status})") print() diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index cc19e94c6b15..d8d9e3816fcb 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -8,6 +8,7 @@ import json import re import sys +from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Optional @@ -27,6 +28,18 @@ ) +def _public_timestamp(value) -> Optional[str]: + if type(value) is not str or len(value) > 64: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None or parsed.utcoffset() is None: + return None + return value + + def _normalize_skills(single_skill=None, skills: Optional[Iterable[str]] = None) -> Optional[List[str]]: if skills is None: if single_skill is None: @@ -197,35 +210,19 @@ def cron_list(show_all: bool = False): print(color("└─────────────────────────────────────────────────────────────────────────┘", Colors.CYAN)) print() - from cron.jobs import effective_job_state + from tools.cronjob_tools import _format_job + + for stored_job in jobs: + # Keep CLI list output on the same bounded projection as the cronjob + # model tool. Stored prompts, targets and execution configuration are + # management detail, not public status fields. + job = _format_job(stored_job) + job_id = job["job_id"] + name = job["name"] + schedule = job["schedule"] + state = job.get("state") or "scheduled" + next_run = job.get("next_run_at") or "?" - for job in jobs: - job_id = job.get("id", "?") - name = job.get("name", "(unnamed)") - schedule = job.get("schedule_display", job.get("schedule", {}).get("value", "?")) - # Derive from the scheduler-honoured flag — never show [paused] when - # enabled=true (half-paused contradiction must not look frozen). - state = effective_job_state(job) - next_run = job.get("next_run_at", "?") - - # `repeat` may be present-but-null in the job record (e.g. a one-shot - # job persisted with "repeat": null), so coalesce to {} rather than - # relying on the dict-default, which only applies to a missing key. - repeat_info = job.get("repeat") or {} - repeat_times = repeat_info.get("times") - repeat_completed = repeat_info.get("completed", 0) - repeat_str = f"{repeat_completed}/{repeat_times}" if repeat_times else "∞" - - # `deliver` may be present-but-null in the job record (same pitfall as - # `repeat` above), so coalesce to the default rather than relying on the - # dict-default, which only applies to a missing key. A null value would - # otherwise reach `", ".join(None)` and crash the whole listing (#32896). - deliver = job.get("deliver") or ["local"] - if isinstance(deliver, str): - deliver = [deliver] - deliver_str = ", ".join(deliver) - - skills = job.get("skills") or ([job["skill"]] if job.get("skill") else []) if state == "paused": status = color("[paused]", Colors.YELLOW) elif state == "completed": @@ -238,38 +235,20 @@ def cron_list(show_all: bool = False): print(f" {color(job_id, Colors.YELLOW)} {status}") print(f" Name: {name}") print(f" Schedule: {schedule}") - print(f" Repeat: {repeat_str}") + print(f" Repeat: {job['repeat']}") print(f" Next run: {next_run}") - print(f" Deliver: {deliver_str}") - if skills: - print(f" Skills: {', '.join(skills)}") - script = job.get("script") - if script: - print(f" Script: {script}") - monitor_source = job.get("monitor_script") or job.get("monitor_url") - if monitor_source: - print(f" Monitor: {monitor_source} (agent runs only on output change)") - mon_state = job.get("monitor_state") or {} - if mon_state.get("last_changed_at"): - print(f" Changed: {mon_state['last_changed_at']}") - if job.get("no_agent"): - print(f" Mode: {color('no-agent', Colors.DIM)} (script stdout delivered directly)") - workdir = job.get("workdir") - if workdir: - print(f" Workdir: {workdir}") - - # Execution history + print(f" Delivery: {job['delivery_kind']}") + print(f" Mode: {job['mode']}") + last_status = job.get("last_status") if last_status: last_run = job.get("last_run_at", "?") if last_status == "ok": status_display = color("ok", Colors.GREEN) elif last_status == "delivery_failed": - # The agent succeeded but the result never reached the user — - # not green, and the detail lives in last_delivery_error - # (last_error is None for these runs). - detail = job.get("last_delivery_error") or "?" - status_display = color(f"delivery_failed: {detail}", Colors.YELLOW) + # Preserve the warning without exposing provider/target detail + # from the model-safe summary projection. + status_display = color("delivery_failed", Colors.YELLOW) else: status_display = color(f"{last_status}: {job.get('last_error', '?')}", Colors.RED) streak = int(job.get("failure_streak") or 0) @@ -281,33 +260,29 @@ def cron_list(show_all: bool = False): if dispatch_line: print(f" Dispatch: {dispatch_line}") - latest_execution = job.get("latest_execution") - if latest_execution: - print( - f" Execution: {latest_execution.get('status', '?')} " - f"{latest_execution.get('id', '?')}" - ) + last_execution = job.get("last_execution") + if isinstance(last_execution, dict) and last_execution.get("status"): + print(f" Execution: {last_execution['status']}") - delivery_err = job.get("last_delivery_error") - if delivery_err: - print(f" {color('⚠ Delivery failed:', Colors.YELLOW)} {delivery_err}") + if job.get("last_delivery_error"): + print(f" {color('⚠ Delivery failed', Colors.YELLOW)}") # A live adapter acked the last send but returned no message_id / # raw_response (Slack/Matrix/Mattermost shape): accepted as delivered, # but say so here rather than only in a WARNING log line. unverified = job.get("last_delivery_unverified") if unverified: - targets = ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) print( f" {color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} " - f"adapter acked {targets} without message_id/raw_response" + "adapter acked without message_id/raw_response" ) fire_err = job.get("last_fire_error") - if isinstance(fire_err, dict) and fire_err.get("detail"): + if isinstance(fire_err, dict): + fire_at = fire_err.get("at") or "?" print( f" {color('⚠ Missed scheduled fire:', Colors.RED)} " - f"{fire_err.get('at', '?')} {fire_err['detail']}" + f"{fire_at}" ) print() @@ -333,9 +308,9 @@ def cron_tick(): except OSError as exc: # tick() now propagates real lock-acquisition failures (EMFILE, # EACCES on open, ...) instead of swallowing them as contention - # (#87644). For the one-shot CLI surface, report cleanly instead of - # dumping a traceback; the gateway ticker loop handles its own retry. - print(color(f"✗ Cron tick failed: {exc}", Colors.RED)) + # (#87644). Public output stays categorical; the gateway log retains + # operator-only diagnostic detail. + print(color("✗ Cron tick failed: tick_failed", Colors.RED)) print(" Check `hermes cron status` and the gateway log for details.") return 1 return 0 @@ -343,7 +318,7 @@ def cron_tick(): def cron_runs(job_id: Optional[str] = None, limit: int = 20): """Show indexed durable cron execution history.""" - from cron.executions import list_executions + from cron.executions import list_executions, receipt_summary records = list_executions(job_id=job_id, limit=limit) if not records: @@ -355,8 +330,16 @@ def cron_runs(job_id: Optional[str] = None, limit: int = 20): f"job={record.get('job_id', '?')} source={record.get('source', '?')} " f"{record.get('claimed_at', '?')}" ) - if record.get("error"): - print(f" {record['error']}") + summary = receipt_summary(str(record.get("id", ""))) + print( + " Receipt: " + f"delivered={summary['delivered']} failed={summary['failed']} " + f"unknown={summary['unknown']} targets_delivered={summary['targets_delivered']}" + ) + if record.get("error_kind"): + # Only the bounded category is durable; raw exception/provider text + # never enters executions.db or this operator surface. + print(f" Failure kind: {record['error_kind']}") _INCIDENT_STATE_COLORS = { @@ -564,13 +547,14 @@ def cron_status(): print(f" PID: {', '.join(map(str, pids))}") last_error = get_ticker_last_error() if last_error: - # Show WHY ticks fail — e.g. a root-rewritten jobs.json - # (PermissionError) that silently locked out the ticker's - # uid for ~14h in the field (#68483), or fd exhaustion - # (EMFILE) that used to stall the scheduler invisibly - # (#87644). - print(color(f" Last tick error: {last_error}", Colors.RED)) if "Permission denied" in last_error: + error_kind = "permission_denied" + elif _cron_is_fd_exhaustion_text(last_error): + error_kind = "fd_exhaustion" + else: + error_kind = "tick_failed" + print(color(f" Last tick error: {error_kind}", Colors.RED)) + if error_kind == "permission_denied": print(color( " Hint: jobs.json may be owned by another user " "(e.g. rewritten by a root `docker exec hermes " @@ -578,7 +562,7 @@ def cron_status(): "gateway user, and prefer `docker exec -u :`.", Colors.YELLOW, )) - elif _cron_is_fd_exhaustion_text(last_error): + elif error_kind == "fd_exhaustion": print(color( " Hint: the ticker hit file-descriptor exhaustion " "(EMFILE). The scheduler now retries with backoff and " @@ -953,11 +937,11 @@ def _stateless_reset() -> None: # success/failure verdict would be a lie (#83340). Report the # background dispatch instead of claiming the run failed. delegation_id = job.get("delegation_id") - if job.get("execution_mode") == "background" or delegation_id: - if delegation_id: - print(f" Running in background (delegation {delegation_id}).") - else: - print(" Running in background.") + if ( + job.get("execution_mode") == "background" + or isinstance(delegation_id, str) and bool(delegation_id) + ): + print(" Running in background.") elif job.get("executed"): outcome = "succeeded" if job.get("execution_success") else "failed" print(f" Ran now: {outcome}.") diff --git a/hermes_cli/web_routers/cron.py b/hermes_cli/web_routers/cron.py index c540c77e1700..3ae069732da0 100644 --- a/hermes_cli/web_routers/cron.py +++ b/hermes_cli/web_routers/cron.py @@ -34,6 +34,7 @@ _run_cron_dashboard_io = late("_run_cron_dashboard_io") _list_cron_jobs_sync = late("_list_cron_jobs_sync") _get_cron_job_sync = late("_get_cron_job_sync") +_get_cron_job_detail_sync = late("_get_cron_job_detail_sync") _list_cron_job_runs_sync = late("_list_cron_job_runs_sync") _create_cron_job_sync = late("_create_cron_job_sync") _update_cron_job_sync = late("_update_cron_job_sync") @@ -48,6 +49,7 @@ _notify_cron_provider_for_profile = late("_notify_cron_provider_for_profile") _call_cron_for_profile = late("_call_cron_for_profile") _raise_if_cron_registration_error = late("_raise_if_cron_registration_error") +_public_cron_job = late("_public_cron_job") load_config = late("load_config") cfg_get = late("cfg_get") @@ -60,6 +62,38 @@ _CRON_FIRE_RETRY_AFTER_SECONDS = 60 +def _public_gateway_fire_body( + status_code: int, + gateway_body: object, + job_id: str, +) -> dict: + body = gateway_body if type(gateway_body) is dict else {} + status = body.get("status") + if ( + type(status_code) is int + and 200 <= status_code < 300 + and type(status) is str + and status in {"accepted", "duplicate"} + ): + return {"status": status, "job_id": job_id} + + if status_code == 400: + error_kind = "invalid_request" + elif status_code in {401, 403}: + error_kind = "authentication_failed" + elif status_code == 409: + error_kind = "claim_conflict" + elif status_code == 503: + error_kind = "gateway_unavailable" + else: + error_kind = "gateway_fire_failed" + return { + "error": error_kind, + "error_kind": error_kind, + "job_id": job_id, + } + + @router.get("/api/cron/jobs") async def list_cron_jobs(profile: str = "all"): return await _run_cron_dashboard_io(_list_cron_jobs_sync, profile) @@ -70,13 +104,20 @@ async def get_cron_job(job_id: str, profile: Optional[str] = None): return await _run_cron_dashboard_io(_get_cron_job_sync, job_id, profile) +@router.get("/api/cron/jobs/{job_id}/detail") +async def get_cron_job_detail(job_id: str, profile: str): + return await _run_cron_dashboard_io( + _get_cron_job_detail_sync, job_id, profile, + ) + + @router.get("/api/cron/jobs/{job_id}/runs") async def list_cron_job_runs(job_id: str, profile: Optional[str] = None, limit: int = 20): return await _run_cron_dashboard_io(_list_cron_job_runs_sync, job_id, profile, limit) @router.post("/api/cron/jobs") -async def create_cron_job(body: CronJobCreate, profile: Optional[str] = None): +async def create_cron_job(body: CronJobCreate, profile: str): return await _run_cron_dashboard_io(_create_cron_job_sync, body, profile) @@ -109,27 +150,27 @@ async def get_cron_delivery_targets(): @router.put("/api/cron/jobs/{job_id}") -async def update_cron_job(job_id: str, body: CronJobUpdate, profile: Optional[str] = None): +async def update_cron_job(job_id: str, body: CronJobUpdate, profile: str): return await _run_cron_dashboard_io(_update_cron_job_sync, job_id, body, profile) @router.post("/api/cron/jobs/{job_id}/pause") -async def pause_cron_job(job_id: str, profile: Optional[str] = None): +async def pause_cron_job(job_id: str, profile: str): return await _run_cron_dashboard_io(_pause_cron_job_sync, job_id, profile) @router.post("/api/cron/jobs/{job_id}/resume") -async def resume_cron_job(job_id: str, profile: Optional[str] = None): +async def resume_cron_job(job_id: str, profile: str): return await _run_cron_dashboard_io(_resume_cron_job_sync, job_id, profile) @router.post("/api/cron/jobs/{job_id}/trigger") -async def trigger_cron_job(job_id: str, profile: Optional[str] = None): +async def trigger_cron_job(job_id: str, profile: str): return await _run_cron_dashboard_io(_trigger_cron_job_sync, job_id, profile) @router.delete("/api/cron/jobs/{job_id}") -async def delete_cron_job(job_id: str, profile: Optional[str] = None): +async def delete_cron_job(job_id: str, profile: str): return await _run_cron_dashboard_io(_delete_cron_job_sync, job_id, profile) @@ -240,25 +281,21 @@ async def cron_fire_webhook(request: Request): return JSONResponse( { "status": "gateway_stopped", - "detail": "gateway deliberately stopped; fire dropped, " - "jobs re-arm on next gateway start", "job_id": job_id, - "profile": profile, }, status_code=200, ) return JSONResponse( { - "error": "gateway unreachable; retry", + "error": "gateway_unavailable", + "error_kind": "gateway_unavailable", "job_id": job_id, - "profile": profile, }, status_code=503, headers={"Retry-After": str(_CRON_FIRE_RETRY_AFTER_SECONDS)}, ) status_code, gateway_body = forwarded - if isinstance(gateway_body, dict): - gateway_body.setdefault("job_id", job_id) + public_body = _public_gateway_fire_body(status_code, gateway_body, job_id) headers = ( # The gateway's own 503s (draining, admission failure) are equally # transient — give the scheduler the same spacing hint. @@ -266,7 +303,7 @@ async def cron_fire_webhook(request: Request): if status_code == 503 else None ) - return JSONResponse(gateway_body, status_code=status_code, headers=headers) + return JSONResponse(public_body, status_code=status_code, headers=headers) @router.get("/api/cron/blueprints") @@ -298,9 +335,9 @@ async def list_cron_blueprints(): f["options"] = deliver_options entries.append(entry) return {"blueprints": entries} - except Exception as e: + except Exception: _log.exception("GET /api/cron/blueprints failed") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="cron_blueprint_list_failed") @router.post("/api/cron/blueprints/instantiate") @@ -330,10 +367,10 @@ async def instantiate_blueprint(body: AutomationBlueprintInstantiate, profile: s # providers on a multi-profile dashboard). Off the event loop — # a Chronos reconcile does file I/O plus NAS network calls. await _run_cron_dashboard_io(_notify_cron_provider_for_profile, profile) - return created + return _public_cron_job(created) except HTTPException: raise except Exception as e: _raise_if_cron_registration_error(e) _log.exception("POST /api/cron/blueprints/instantiate failed") - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code=400, detail="cron_create_failed") from e diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index bc0db86e04ea..770678684351 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -43,6 +43,7 @@ import tempfile import threading import time +import unicodedata import urllib.error import urllib.parse import zipfile @@ -13241,10 +13242,276 @@ def _find_cron_job_profile(job_id: str) -> Optional[str]: return None +_PUBLIC_CRON_JOB_FIELDS = frozenset({ + "id", + "name", + "schedule", + "schedule_display", + "repeat", + "enabled", + "state", + "last_run_at", + "next_run_at", + "last_status", + "last_error", + "last_delivery_error", + "last_fire_error", +}) + +_PUBLIC_CRON_JOB_DETAIL_STRING_LIMITS = { + "prompt": 1_000_000, + "script": 4096, + "deliver": 1024, + "model": 512, + "provider": 256, + "base_url": 4096, + "workdir": 4096, + "monitor_script": 4096, + "monitor_url": 4096, + "reasoning_effort": 64, +} + + +def _public_cron_timestamp(value: Any) -> Optional[str]: + """Return one bounded timezone-aware ISO timestamp, otherwise ``None``.""" + if type(value) is not str or len(value) > 64: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None or parsed.utcoffset() is None: + return None + return value + + +_PUBLIC_CRON_RUN_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") + + +def _public_cron_run_time(value: Any) -> Optional[float]: + """Return one finite non-negative epoch value, otherwise ``None``.""" + if type(value) not in {int, float}: + return None + parsed = float(value) + if not math.isfinite(parsed) or parsed < 0: + return None + return parsed + + +def _public_cron_run(run: Any, *, now: float) -> Optional[Dict[str, Any]]: + """Project a session-backed cron run onto its bounded public shape.""" + if type(run) is not dict: + return None + run_id = run.get("id") + if type(run_id) is not str or not _PUBLIC_CRON_RUN_ID_RE.fullmatch(run_id): + return None + + started_at = _public_cron_run_time(run.get("started_at")) + ended_raw = run.get("ended_at") + ended_at = _public_cron_run_time(ended_raw) + last_active = _public_cron_run_time(run.get("last_active")) + + if ended_raw is None: + status = "running" + else: + reason = run.get("end_reason") + if type(reason) is not str: + reason = None + if reason in {"completed", "success", "agent_close"}: + status = "completed" + elif reason in {"failed", "error"}: + status = "failed" + elif reason in {"cancelled", "interrupted"}: + status = "cancelled" + elif reason == "timeout": + status = "timeout" + else: + status = "ended" + + is_active = ( + ended_raw is None + and last_active is not None + and math.isfinite(now) + and 0 <= now - last_active < 300 + ) + archived_raw = run.get("archived") + archived = archived_raw is True or ( + type(archived_raw) is int and archived_raw == 1 + ) + return { + "id": run_id, + "status": status, + "started_at": started_at, + "ended_at": ended_at, + "last_active": last_active, + "is_active": is_active, + "archived": archived, + } + + +def _public_cron_text(value: Any, *, limit: int) -> Optional[str]: + if type(value) is not str or not value or len(value) > limit: + return None + if any(unicodedata.category(char).startswith("C") for char in value): + return None + return value + + +def _public_cron_string_list(value: Any, *, item_limit: int = 256) -> List[str]: + if type(value) not in {list, tuple} or len(value) > 256: + return [] + return [ + item + for item in value + if _public_cron_text(item, limit=item_limit) is not None + ] + + +def _public_cron_job(job: Any) -> Dict[str, Any]: + """Return the bounded summary contract used by lists and mutations.""" + if type(job) is not dict: + raise TypeError("cron job projection requires an object") + public = { + key: job[key] + for key in _PUBLIC_CRON_JOB_FIELDS + if key in job + } + for text_field, limit in ( + ("id", 128), + ("name", 256), + ("schedule_display", 256), + ): + if text_field in public: + public[text_field] = _public_cron_text(public[text_field], limit=limit) + if "schedule" in public: + schedule = public["schedule"] + if type(schedule) is dict: + public["schedule"] = { + "kind": _public_cron_text(schedule.get("kind"), limit=32), + "expr": _public_cron_text(schedule.get("expr"), limit=256), + "run_at": _public_cron_timestamp(schedule.get("run_at")), + "display": _public_cron_text(schedule.get("display"), limit=256), + } + else: + public["schedule"] = None + if "repeat" in public: + repeat = public["repeat"] + if type(repeat) is dict: + public["repeat"] = { + key: value if type(value) is int and value >= 0 else None + for key in ("times", "completed") + if (value := repeat.get(key)) is not None + } + else: + public["repeat"] = None + if "enabled" in public and type(public["enabled"]) is not bool: + public["enabled"] = False + for category_field in ("state", "last_status"): + if category_field in public: + value = public[category_field] + public[category_field] = ( + value + if type(value) is str + and re.fullmatch(r"[a-z][a-z0-9_]{0,31}", value) + else None + ) + if public.get("last_error") is not None: + public["last_error"] = "run_failed" + if public.get("last_delivery_error") is not None: + public["last_delivery_error"] = "delivery_failed" + for timestamp_field in ("last_run_at", "next_run_at"): + if timestamp_field in public: + public[timestamp_field] = _public_cron_timestamp(public[timestamp_field]) + fire_error = public.get("last_fire_error") + if type(fire_error) is dict: + public["last_fire_error"] = { + "at": _public_cron_timestamp(fire_error.get("at")), + "error_kind": "fire_forward_failed", + } + elif fire_error is not None: + public["last_fire_error"] = None + + deliver = job.get("deliver") + if type(deliver) is str and deliver in {"local", "origin", "all"}: + public["delivery_kind"] = deliver + elif type(deliver) is str and deliver: + public["delivery_kind"] = "external" + else: + public["delivery_kind"] = "local" + if any( + type(job.get(key)) is str and bool(job.get(key)) + for key in ("monitor_script", "monitor_url") + ): + public["mode"] = "monitor" + elif job.get("no_agent") is True: + public["mode"] = "script" + else: + public["mode"] = "agent" + public["skill_count"] = min( + len(_public_cron_string_list(job.get("skills"))), 9999, + ) + public["toolset_count"] = min( + len(_public_cron_string_list(job.get("enabled_toolsets"))), 9999, + ) + public["model_configured"] = any( + type(job.get(key)) is str and bool(job.get(key)) + for key in ("model", "provider", "base_url") + ) + return public + + +def _public_cron_job_for_profile(job: Any, profile: str) -> Dict[str, Any]: + """Add only the validated route identity to a bounded job summary.""" + from hermes_cli import profiles as profiles_mod + + profile_name = profiles_mod.normalize_profile_name(profile) + profiles_mod.validate_profile_name(profile_name) + public = _public_cron_job(job) + public["profile"] = profile_name + public["profile_name"] = profile_name + public["is_default_profile"] = profile_name == "default" + return public + + +def _public_cron_detail_prompt(value: Any, *, limit: int) -> Optional[str]: + if type(value) is not str or not value or len(value) > limit: + return None + for char in value: + if unicodedata.category(char).startswith("C") and char not in {"\n", "\r", "\t"}: + return None + return value + + +def _public_cron_job_detail(job: Any) -> Dict[str, Any]: + """Return explicit editable config without profile/runtime internals.""" + if type(job) is not dict: + raise TypeError("cron job detail projection requires an object") + detail = _public_cron_job(job) + for key, limit in _PUBLIC_CRON_JOB_DETAIL_STRING_LIMITS.items(): + if key in job: + projector = _public_cron_detail_prompt if key == "prompt" else _public_cron_text + detail[key] = ( + projector(job[key], limit=limit) + if job[key] is not None + else None + ) + for key in ("skills", "context_from", "enabled_toolsets"): + if key in job: + detail[key] = _public_cron_string_list(job[key]) + if "no_agent" in job: + detail["no_agent"] = job["no_agent"] if type(job["no_agent"]) is bool else False + if "continuity" in job: + detail["continuity"] = job["continuity"] if type(job["continuity"]) is bool else False + return detail + + def _list_cron_jobs_sync(profile: str = "all"): requested = (profile or "all").strip() if requested.lower() != "all": - return _call_cron_for_profile(requested, "list_jobs", True) + return [ + _public_cron_job_for_profile(job, requested) + for job in _call_cron_for_profile(requested, "list_jobs", True) + ] jobs: List[Dict[str, Any]] = [] for item in _cron_profile_dicts(): @@ -13252,7 +13519,10 @@ def _list_cron_jobs_sync(profile: str = "all"): if not name: continue try: - jobs.extend(_call_cron_for_profile(name, "list_jobs", True)) + jobs.extend( + _public_cron_job_for_profile(job, name) + for job in _call_cron_for_profile(name, "list_jobs", True) + ) except Exception: _log.exception("Failed to list cron jobs for profile %s", name) return jobs @@ -13287,6 +13557,7 @@ def _raise_if_cron_registration_error(e: Exception) -> None: from hermes_cli.web_routers.cron import ( # noqa: E402,F401 — legacy re-exports; tests call these via web_server. list_cron_jobs, get_cron_job, + get_cron_job_detail, list_cron_job_runs, create_cron_job, get_cron_delivery_targets, @@ -13308,7 +13579,17 @@ def _get_cron_job_sync(job_id: str, profile: Optional[str] = None): job = _call_cron_for_profile(selected, "get_job", job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") - return job + return _public_cron_job_for_profile(job, selected) + + +def _get_cron_job_detail_sync(job_id: str, profile: str): + selected = (profile or "").strip() + if not selected or selected.lower() == "all": + raise HTTPException(status_code=400, detail="cron_detail_profile_required") + job = _call_cron_for_profile(selected, "get_job", job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + return _public_cron_job_detail(job) @@ -13320,8 +13601,9 @@ def _list_cron_job_runs_sync(job_id: str, profile: Optional[str] = None, limit: ``cron_{job_id}_{timestamp}`` (see cron/scheduler.run_job). A job's history is therefore every session whose id carries that prefix; ``source='cron'`` narrows it and the id prefix binds it to this job. Powers the run-history - list under each job in the desktop cron detail. Same row shape as - ``/api/sessions`` so the frontend can reuse SessionInfo. + list under each job in the desktop cron detail. Session rows are projected + onto a dedicated bounded shape; prompts, previews, paths, provider/runtime + details, and unknown future fields never cross this public boundary. Backed by ``SessionDB.list_cron_job_runs`` — a bounded ``[prefix, hi)`` id-range scan, not the compression-chain CTE used for the recents list, @@ -13343,16 +13625,13 @@ def _list_cron_job_runs_sync(job_id: str, profile: Optional[str] = None, limit: db = _open_session_db_for_profile(selected, read_only=True) try: - runs = db.list_cron_job_runs(canonical, limit=limit_n, offset=0) + rows = db.list_cron_job_runs(canonical, limit=limit_n, offset=0) now = time.time() - for s in runs: - s["is_active"] = ( - s.get("ended_at") is None - and (now - s.get("last_active", s.get("started_at", 0))) < 300 - ) - s["archived"] = bool(s.get("archived")) - if selected: - s["profile"] = selected + runs = [ + public + for row in rows + if (public := _public_cron_run(row, now=now)) is not None + ] return {"runs": runs, "limit": limit_n} finally: db.close() @@ -13360,9 +13639,18 @@ def _list_cron_job_runs_sync(job_id: str, profile: Optional[str] = None, limit: +def _require_concrete_cron_profile(profile: Optional[str]) -> str: + selected = (profile or "").strip() + if not selected or selected.lower() == "all": + raise HTTPException(status_code=400, detail="cron_mutation_profile_required") + profile_name, _profile_home = _cron_profile_home(selected) + return profile_name + + def _create_cron_job_sync(body: CronJobCreate, profile: Optional[str] = None): try: - profile_name, profile_home = _cron_profile_home(profile) + profile_name = _require_concrete_cron_profile(profile) + _profile_name, profile_home = _cron_profile_home(profile_name) script = _normalize_dashboard_cron_script(body.script, profile_home) skills = _cron_string_list(body.skills) context_from = _cron_string_list(body.context_from) @@ -13374,7 +13662,7 @@ def _create_cron_job_sync(body: CronJobCreate, profile: Optional[str] = None): "script": script, "no_agent": no_agent, }) - return _mutate_cron_for_profile( + job = _mutate_cron_for_profile( profile_name, "create_job", prompt=body.prompt or "", @@ -13391,12 +13679,13 @@ def _create_cron_job_sync(body: CronJobCreate, profile: Optional[str] = None): workdir=_cron_optional_text(body.workdir), no_agent=no_agent, ) + return _public_cron_job(job) except HTTPException: raise except Exception as e: _raise_if_cron_registration_error(e) _log.exception("POST /api/cron/jobs failed") - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code=400, detail="cron_create_failed") from e @@ -13404,9 +13693,7 @@ def _create_cron_job_sync(body: CronJobCreate, profile: Optional[str] = None): def _update_cron_job_sync(job_id: str, body: CronJobUpdate, profile: Optional[str] = None): - selected = profile or _find_cron_job_profile(job_id) - if not selected: - raise HTTPException(status_code=404, detail="Job not found") + selected = _require_concrete_cron_profile(profile) try: profile_name, profile_home = _cron_profile_home(selected) existing = _call_cron_for_profile(profile_name, "get_job", job_id) @@ -13431,42 +13718,37 @@ def _update_cron_job_sync(job_id: str, body: CronJobUpdate, profile: Optional[st except HTTPException: raise except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + detail = "cron_update_invalid_id" if "id" in body.updates else "cron_update_failed" + raise HTTPException(status_code=400, detail=detail) from exc if not job: raise HTTPException(status_code=404, detail="Job not found") - return job + return _public_cron_job(job) def _pause_cron_job_sync(job_id: str, profile: Optional[str] = None): - selected = profile or _find_cron_job_profile(job_id) - if not selected: - raise HTTPException(status_code=404, detail="Job not found") + selected = _require_concrete_cron_profile(profile) job = _mutate_cron_for_profile(selected, "pause_job", job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") - return job + return _public_cron_job(job) def _resume_cron_job_sync(job_id: str, profile: Optional[str] = None): - selected = profile or _find_cron_job_profile(job_id) - if not selected: - raise HTTPException(status_code=404, detail="Job not found") + selected = _require_concrete_cron_profile(profile) job = _mutate_cron_for_profile(selected, "resume_job", job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") - return job + return _public_cron_job(job) def _trigger_cron_job_sync(job_id: str, profile: Optional[str] = None): - selected = profile or _find_cron_job_profile(job_id) - if not selected: - raise HTTPException(status_code=404, detail="Job not found") + selected = _require_concrete_cron_profile(profile) job = _call_cron_for_profile(selected, "resolve_job_ref", job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") @@ -13479,34 +13761,32 @@ def _trigger_cron_job_sync(job_id: str, profile: Optional[str] = None): ran = _fire_cron_job_for_profile(selected, job["id"], force=force) refreshed = _call_cron_for_profile(selected, "get_job", job["id"]) if refreshed and refreshed.get("last_run_at") != job.get("last_run_at"): - return refreshed + return _public_cron_job(refreshed) if not ran: raise HTTPException( status_code=409, detail="Job is already running or was claimed by another scheduler", ) if refreshed: - return refreshed + return _public_cron_job(refreshed) # A one-shot may remove itself after exhausting repeat=1. Keep the response # shape compatible without inventing an outcome that is no longer present # in the job store; authoritative list refresh removes the completed row. - return { + return _public_cron_job({ **job, "enabled": False, "state": "completed", - } + }) def _delete_cron_job_sync(job_id: str, profile: Optional[str] = None): - selected = profile or _find_cron_job_profile(job_id) - if not selected: - raise HTTPException(status_code=404, detail="Job not found") + selected = _require_concrete_cron_profile(profile) try: removed = _mutate_cron_for_profile(selected, "remove_job", job_id) except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + raise HTTPException(status_code=400, detail="cron_delete_failed") from exc if not removed: raise HTTPException(status_code=404, detail="Job not found") return {"ok": True} diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py index a09f715c901b..e0c9737b7dcd 100644 --- a/plugins/platforms/matrix/adapter.py +++ b/plugins/platforms/matrix/adapter.py @@ -134,6 +134,9 @@ class _TrustStateStub: # type: ignore[no-redef] MessageType, ProcessingOutcome, SendResult, + TransportReceipt, + TransportTarget, + normalize_transport_provider_message_id, resolve_proxy_url, proxy_kwargs_for_aiohttp, _ssrf_redirect_guard, @@ -2210,6 +2213,68 @@ async def disconnect(self) -> None: logger.info("Matrix: disconnected") + def plan_transport_text(self, content: str) -> list[str]: + """Expose Matrix's exact deterministic text chunks before dispatch.""" + formatted = self.format_message(content) + return list(self.truncate_message(formatted, self.max_message_length)) + + @staticmethod + def _transport_receipt_targets( + chat_id: str, + metadata: Optional[Dict[str, Any]], + ) -> tuple[TransportTarget, TransportTarget]: + """Return the planned target and the exact routed Matrix target.""" + if metadata is None: + metadata = {} + elif type(metadata) is not dict: + raise TypeError("Matrix receipt metadata must be an object") + if type(chat_id) is not str: + raise TypeError("Matrix receipt chat_id must be a string") + route_thread_raw = metadata.get("thread_id") + if route_thread_raw is not None and type(route_thread_raw) not in {str, int}: + raise TypeError("Matrix receipt thread_id must be a string or integer") + route_thread = ( + str(route_thread_raw) if route_thread_raw is not None else None + ) + requested_identity = metadata.get( + "_transport_receipt_requested_target" + ) + if requested_identity is not None and type(requested_identity) is not dict: + raise TypeError("Matrix receipt requested target must be a mapping") + if requested_identity is not None: + requested_thread_raw = requested_identity.get("thread_id") + if ( + requested_thread_raw is not None + and type(requested_thread_raw) not in {str, int} + ): + raise TypeError( + "Matrix receipt requested thread_id must be a string or integer" + ) + requested_platform = requested_identity.get("platform", "matrix") + requested_chat_id = requested_identity.get("chat_id", chat_id) + if type(requested_platform) is not str or type(requested_chat_id) is not str: + raise TypeError("Matrix receipt requested target fields must be strings") + requested = TransportTarget( + platform=requested_platform, + chat_id=requested_chat_id, + thread_id=( + str(requested_thread_raw) + if requested_thread_raw is not None else None + ), + ) + else: + requested = TransportTarget( + platform="matrix", + chat_id=chat_id, + thread_id=route_thread, + ) + actual = TransportTarget( + platform="matrix", + chat_id=chat_id, + thread_id=route_thread, + ) + return requested, actual + async def send( self, chat_id: str, @@ -2218,14 +2283,30 @@ async def send( metadata: Optional[Dict[str, Any]] = None, ) -> SendResult: """Send a message to a Matrix room.""" - + try: + requested_target, actual_target = self._transport_receipt_targets( + chat_id, metadata + ) + except (TypeError, ValueError): + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) + if type(content) is not str: + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) if not content: return SendResult(success=True) - formatted = self.format_message(content) - chunks = self.truncate_message(formatted, self.max_message_length) - + chunks = self.plan_transport_text(content) last_event_id = None + receipts = [] for i, chunk in enumerate(chunks): msg_content = self._build_text_message_content(chunk) @@ -2240,39 +2321,42 @@ async def send( ), timeout=45, ) - last_event_id = str(event_id) + last_event_id = normalize_transport_provider_message_id(event_id) + if last_event_id is None: + raise ValueError("Matrix provider acknowledgement id is invalid") + receipts.append(TransportReceipt( + outcome="delivered", provider_message_id=last_event_id, + requested_target=requested_target, actual_target=actual_target, + component="text", ordinal=i, + )) logger.info("Matrix: sent event %s to %s", last_event_id, chat_id) - except Exception as exc: - # On E2EE errors, retry after sharing keys. - if self._encryption and getattr(self._client, "crypto", None): - try: - await self._client.crypto.share_keys() - event_id = await asyncio.wait_for( - self._client.send_message_event( - RoomID(chat_id), - EventType.ROOM_MESSAGE, - msg_content, - ), - timeout=45, - ) - last_event_id = str(event_id) - logger.info( - "Matrix: sent event %s to %s (after key share)", - last_event_id, - chat_id, - ) - continue - except Exception as retry_exc: - logger.error( - "Matrix: failed to send to %s after retry: %s", - chat_id, - retry_exc, - ) - return SendResult(success=False, error=str(retry_exc)) - logger.error("Matrix: failed to send to %s: %s", chat_id, exc) - return SendResult(success=False, error=str(exc)) + except Exception: + receipts.append(TransportReceipt( + outcome="unknown", + requested_target=requested_target, + component="text", + ordinal=i, + )) + logger.warning( + "Matrix: delivery outcome is unknown for %s; suppressing retry", + chat_id, + ) + return SendResult( + success=False, + error="Matrix delivery outcome is unknown", + error_kind="unknown", + receipts=tuple(receipts), + retryable=False, + ) - return SendResult(success=True, message_id=last_event_id) + # ``send_message_event`` returning an event id is Matrix's explicit + # acknowledgement. Preserve the legacy message_id separately while + # making the exact room target available to conservative cron callers. + return SendResult( + success=True, + message_id=last_event_id, + receipts=tuple(receipts), + ) async def get_chat_info(self, chat_id: str) -> Dict[str, Any]: """Return room name and type (dm/group).""" @@ -2904,6 +2988,59 @@ async def _upload_and_send( voice_metadata: Optional[Dict[str, Any]] = None, ) -> SendResult: """Upload bytes to Matrix and send as a media message.""" + if ( + type(room_id) is not str + or type(data) is not bytes + or type(filename) is not str + or type(content_type) is not str + or type(msgtype) is not str + or (caption is not None and type(caption) is not str) + or (metadata is not None and type(metadata) is not dict) + ): + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) + receipt_binding = None + receipt_metadata = metadata if metadata is not None else {} + if ( + "_transport_receipt_component" in receipt_metadata + or "_transport_receipt_ordinal" in receipt_metadata + ): + receipt_component = receipt_metadata.get("_transport_receipt_component") + if type(receipt_component) is not str or receipt_component != "media": + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) + receipt_ordinal = receipt_metadata.get("_transport_receipt_ordinal") + if type(receipt_ordinal) is not int or receipt_ordinal < 0: + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) + try: + requested_target, actual_target = self._transport_receipt_targets( + room_id, metadata + ) + except (TypeError, ValueError): + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) + receipt_binding = ( + requested_target, + actual_target, + receipt_ordinal, + ) if len(data) > self._max_media_bytes: return SendResult( success=False, @@ -2978,9 +3115,49 @@ async def _upload_and_send( EventType.ROOM_MESSAGE, msg_content, ) - return SendResult(success=True, message_id=str(event_id)) - except Exception as exc: - return SendResult(success=False, error=str(exc)) + provider_message_id = normalize_transport_provider_message_id(event_id) + if provider_message_id is None: + raise ValueError("Matrix provider acknowledgement id is invalid") + if receipt_binding is not None: + requested_target, actual_target, receipt_ordinal = receipt_binding + receipt = TransportReceipt( + outcome="delivered", + provider_message_id=provider_message_id, + requested_target=requested_target, + actual_target=actual_target, + component="media", + ordinal=receipt_ordinal, + ) + return SendResult( + success=True, + message_id=provider_message_id, + receipt=receipt, + receipts=(receipt,), + ) + return SendResult(success=True, message_id=provider_message_id) + except Exception: + if receipt_binding is not None: + requested_target, _actual_target, receipt_ordinal = receipt_binding + receipt = TransportReceipt( + outcome="unknown", + requested_target=requested_target, + component="media", + ordinal=receipt_ordinal, + ) + return SendResult( + success=False, + error="Matrix media delivery outcome is unknown", + error_kind="unknown", + receipt=receipt, + receipts=(receipt,), + retryable=False, + ) + return SendResult( + success=False, + error="Matrix media delivery outcome is unknown", + error_kind="unknown", + retryable=False, + ) async def _send_local_file( self, diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index d50f631c272c..5cc15fdfdb18 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -222,6 +222,9 @@ class _MockContextTypes: MessageType, ProcessingOutcome, SendResult, + TransportReceipt, + TransportTarget, + normalize_transport_provider_message_id, classify_send_error, cache_image_from_bytes_async, cache_audio_from_bytes_async, @@ -601,6 +604,10 @@ class _PollingLifecycleAbort(RuntimeError): """Internal control flow for polling startup fenced by teardown.""" +class _InvalidTransportReceiptMetadata(ValueError): + """Scheduler-owned receipt metadata failed before any provider side effect.""" + + class TelegramAdapter(BasePlatformAdapter): """ Telegram bot adapter. @@ -639,6 +646,71 @@ class TelegramAdapter(BasePlatformAdapter): _RECONNECT_WAIT_SECONDS = 15.0 _RECONNECT_POLL_INTERVAL = 0.5 + @staticmethod + def _transport_identity_text(value: Any, *, field: str) -> str: + if type(value) is str: + return value + if type(value) is int: + return str(value) + raise _InvalidTransportReceiptMetadata( + f"transport receipt {field} must be a string or integer" + ) + + @classmethod + def _validated_transport_metadata( + cls, chat_id: Any, metadata: Optional[Dict[str, Any]], + ) -> Dict[str, Any]: + if metadata is None: + return {} + if type(metadata) is not dict: + raise _InvalidTransportReceiptMetadata( + "transport receipt metadata must be an object" + ) + routed_chat = cls._transport_identity_text(chat_id, field="chat_id") + validated = dict(metadata) + for key in ( + "thread_id", + "message_thread_id", + "direct_messages_topic_id", + "telegram_direct_messages_topic_id", + "telegram_reply_to_message_id", + ): + raw_value = metadata.get(key) + if raw_value is not None: + validated[key] = cls._transport_identity_text( + raw_value, field=key, + ) + requested = metadata.get("_transport_receipt_requested_target") + if requested is None: + return validated + if type(requested) is not dict: + raise _InvalidTransportReceiptMetadata( + "transport receipt requested target must be an object" + ) + platform = requested.get("platform", "telegram") + if type(platform) is not str: + raise _InvalidTransportReceiptMetadata( + "transport receipt platform must be a string" + ) + requested_chat = requested.get("chat_id", routed_chat) + requested_thread = requested.get("thread_id") + target = TransportTarget( + platform=platform, + chat_id=cls._transport_identity_text( + requested_chat, field="chat_id", + ), + thread_id=( + cls._transport_identity_text(requested_thread, field="thread_id") + if requested_thread is not None else None + ), + ) + validated["_transport_receipt_requested_target"] = { + "platform": target.platform, + "chat_id": target.chat_id, + "thread_id": target.thread_id, + } + return validated + # Telegram's edit_message applies MarkdownV2 formatting only on the # finalize=True path. Without this flag, stream_consumer._send_or_edit # short-circuits when the raw text is unchanged between the last streamed @@ -1847,8 +1919,18 @@ async def _send_with_dm_topic_reply_anchor_retry( reply_to_message_id: Optional[int], media_label: str, reset_media: Optional[Any] = None, + actual_thread_out: Optional[Dict[str, Optional[str]]] = None, ) -> Any: """Retry stale private-topic media replies once without the topic anchor.""" + if actual_thread_out is not None: + routed_thread = ( + send_kwargs.get("message_thread_id") + if send_kwargs.get("message_thread_id") is not None + else send_kwargs.get("direct_messages_topic_id") + ) + actual_thread_out["thread_id"] = ( + str(routed_thread) if routed_thread is not None else None + ) try: return await send_fn(**send_kwargs) except Exception as send_err: @@ -1871,8 +1953,146 @@ async def _send_with_dm_topic_reply_anchor_retry( retry_kwargs["reply_to_message_id"] = None retry_kwargs.pop("message_thread_id", None) retry_kwargs.pop("direct_messages_topic_id", None) + if actual_thread_out is not None: + actual_thread_out["thread_id"] = None return await send_fn(**retry_kwargs) + def _transport_media_receipt_plan( + self, + chat_id: str, + metadata: Optional[Dict[str, Any]], + routed_thread: Optional[Any], + ) -> Optional[tuple[TransportTarget, int]]: + """Validate scheduler-owned media receipt metadata before dispatch.""" + if metadata is None: + metadata = {} + elif type(metadata) is not dict: + raise _InvalidTransportReceiptMetadata( + "transport receipt metadata must be an object" + ) + has_component = "_transport_receipt_component" in metadata + has_ordinal = "_transport_receipt_ordinal" in metadata + if not has_component and not has_ordinal: + return None + component = metadata.get("_transport_receipt_component") + if type(component) is not str or component != "media": + raise _InvalidTransportReceiptMetadata( + "transport receipt component must be media" + ) + ordinal = metadata.get("_transport_receipt_ordinal") + if type(ordinal) is not int or ordinal < 0: + raise _InvalidTransportReceiptMetadata( + "transport receipt ordinal must be a non-negative integer" + ) + + requested_raw = metadata.get("_transport_receipt_requested_target") + try: + if requested_raw is None: + requested_target = TransportTarget( + platform="telegram", + chat_id=self._transport_identity_text(chat_id, field="chat_id"), + thread_id=( + self._transport_identity_text(routed_thread, field="thread_id") + if routed_thread is not None else None + ), + ) + else: + if type(requested_raw) is not dict: + raise TypeError( + "transport receipt requested target must be an object" + ) + requested_target = TransportTarget( + platform=requested_raw.get("platform", "telegram"), + chat_id=requested_raw.get( + "chat_id", + self._transport_identity_text(chat_id, field="chat_id"), + ), + thread_id=requested_raw.get("thread_id"), + ) + except (TypeError, ValueError) as exc: + raise _InvalidTransportReceiptMetadata( + "transport receipt requested target is invalid" + ) from exc + return requested_target, ordinal + + def _transport_media_receipt_context( + self, + chat_id: str, + metadata: Optional[Dict[str, Any]], + thread_kwargs: Dict[str, Any], + ) -> tuple[ + Optional[tuple[TransportTarget, int]], + Dict[str, Optional[str]], + ]: + routed_thread = ( + thread_kwargs.get("message_thread_id") + if thread_kwargs.get("message_thread_id") is not None + else thread_kwargs.get("direct_messages_topic_id") + ) + return ( + self._transport_media_receipt_plan(chat_id, metadata, routed_thread), + {}, + ) + + @staticmethod + def _invalid_transport_media_receipt_result() -> SendResult: + return SendResult( + success=False, + error="Invalid transport receipt metadata", + error_kind="invalid_transport_receipt", + retryable=False, + ) + + @staticmethod + def _transport_media_unknown_result( + plan: tuple[TransportTarget, int], + ) -> SendResult: + requested_target, ordinal = plan + receipt = TransportReceipt( + outcome="unknown", + requested_target=requested_target, + component="media", + ordinal=ordinal, + ) + return SendResult( + success=False, + error="Telegram media delivery outcome is unknown", + error_kind="unknown", + receipt=receipt, + retryable=False, + ) + + @staticmethod + def _transport_media_send_result( + msg: Any, + chat_id: str, + plan: Optional[tuple[TransportTarget, int]], + actual_thread: Dict[str, Optional[str]], + ) -> SendResult: + provider_id = normalize_transport_provider_message_id( + getattr(msg, "message_id", None) + ) + if plan is not None and provider_id is None: + return TelegramAdapter._transport_media_unknown_result(plan) + if plan is None: + return SendResult(success=True, message_id=provider_id) + requested_target, ordinal = plan + receipt = TransportReceipt( + outcome="delivered", + provider_message_id=provider_id, + requested_target=requested_target, + actual_target=TransportTarget( + platform="telegram", + chat_id=TelegramAdapter._transport_identity_text( + chat_id, field="chat_id", + ), + thread_id=actual_thread.get("thread_id"), + ), + component="media", + ordinal=ordinal, + ) + return SendResult(success=True, message_id=provider_id, receipt=receipt) + def _fallback_ips(self) -> list[str]: """Return validated fallback IPs from config (populated by _apply_env_overrides).""" configured = self.config.extra.get("fallback_ips", []) if getattr(self.config, "extra", None) else [] @@ -5458,6 +5678,21 @@ def _should_thread_reply(self, reply_to: Optional[str], chunk_index: int) -> boo else: # "first" (default) return chunk_index == 0 + def plan_transport_text(self, content: str) -> list[str]: + """Expose Telegram's exact MarkdownV2 split before provider dispatch.""" + formatted = self.format_message(content) + chunks = self.truncate_message( + formatted, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len, + ) + if len(chunks) > 1: + chunks = [ + _separate_chunk_indicator_from_fence( + re.sub(r" \((\d+)/(\d+)\)$", r" \\(\1/\2\\)", chunk) + ) + for chunk in chunks + ] + return list(chunks) + async def send( self, chat_id: str, @@ -5486,6 +5721,13 @@ async def send( if getattr(self, "_send_path_degraded", False): return SendResult(success=False, error="send_path_degraded", retryable=True) + try: + metadata = self._validated_transport_metadata(chat_id, metadata) + except (TypeError, ValueError): + return self._invalid_transport_media_receipt_result() + if type(content) is not str: + return self._invalid_transport_media_receipt_result() + # Skip whitespace-only text to prevent Telegram 400 empty-text errors. if not content or not content.strip(): return SendResult(success=True, message_id=None) @@ -5513,25 +5755,37 @@ async def send( pass # Typing failures are non-fatal return rich_result - # Format and split message if needed - formatted = self.format_message(content) - chunks = self.truncate_message( - formatted, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len, - ) - if len(chunks) > 1: - # truncate_message appends a raw " (1/2)" suffix. Escape the - # MarkdownV2-special parentheses so Telegram doesn't reject the - # chunk and fall back to plain text. - chunks = [ - _separate_chunk_indicator_from_fence( - re.sub(r" \((\d+)/(\d+)\)$", r" \\(\1/\2\\)", chunk) - ) - for chunk in chunks - ] + chunks = self.plan_transport_text(content) message_ids = [] + receipts = [] + receipt_bound_send = ( + "_transport_receipt_requested_target" in (metadata or {}) + ) thread_id = self._metadata_thread_id(metadata) requested_thread_id = self._message_thread_id_for_send(thread_id) + receipt_requested = metadata.get("_transport_receipt_requested_target") or {} + receipt_requested_thread = ( + receipt_requested["thread_id"] + if receipt_requested.get("thread_id") + else ( + None + if receipt_requested + else (str(requested_thread_id) if requested_thread_id is not None else None) + ) + ) + receipt_requested_target = TransportTarget( + platform=receipt_requested.get("platform") or "telegram", + chat_id=( + receipt_requested.get("chat_id") + or self._transport_identity_text(chat_id, field="chat_id") + ), + thread_id=receipt_requested_thread, + ) + # Initialize for type-safety and for the degenerate empty-chunk + # path; each successful chunk then replaces it with its actual + # route before the receipt is created. + effective_thread_id = requested_thread_id used_thread_fallback = False try: @@ -5590,7 +5844,11 @@ async def send( if used_thread_fallback and thread_kwargs.get("message_thread_id") is not None: thread_kwargs = dict(thread_kwargs) thread_kwargs["message_thread_id"] = None - effective_thread_id = thread_kwargs.get("message_thread_id") + effective_thread_id = ( + thread_kwargs.get("message_thread_id") + if thread_kwargs.get("message_thread_id") is not None + else thread_kwargs.get("direct_messages_topic_id") + ) msg = None for _send_attempt in range(3): @@ -5609,6 +5867,16 @@ async def send( except Exception as md_error: # Markdown parsing failed, try plain text if "parse" in str(md_error).lower() or "markdown" in str(md_error).lower(): + if receipt_bound_send: + if _BadReq and isinstance(md_error, _BadReq): + return SendResult( + success=False, + error="telegram_markdown_parse_failed", + retryable=False, + error_kind="provider_rejected", + receipts=tuple(receipts), + ) + raise logger.warning("[%s] MarkdownV2 parse failed, falling back to plain text: %s", self.name, md_error) plain_chunk = _strip_mdv2(chunk) msg = await self._bot.send_message( @@ -5698,7 +5966,11 @@ async def send( reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode, ) - effective_thread_id = thread_kwargs.get("message_thread_id") + effective_thread_id = ( + thread_kwargs.get("message_thread_id") + if thread_kwargs.get("message_thread_id") is not None + else thread_kwargs.get("direct_messages_topic_id") + ) continue # Other BadRequest errors are permanent — don't retry raise @@ -5760,7 +6032,44 @@ async def send( await asyncio.sleep(wait) continue raise - message_ids.append(str(msg.message_id)) + provider_message_id = normalize_transport_provider_message_id( + getattr(msg, "message_id", None) + ) + if provider_message_id is None: + receipts.append(TransportReceipt( + outcome="unknown", + requested_target=receipt_requested_target, + component="text", + ordinal=i, + )) + return SendResult( + success=False, + error="Telegram delivery acknowledgement is invalid", + error_kind="unknown", + receipts=tuple(receipts), + retryable=False, + ) + message_ids.append(provider_message_id) + # Each Bot API Message is an independent acknowledgement. Do + # not collapse a split response into its final message id. + receipts.append(TransportReceipt( + outcome="delivered", + provider_message_id=provider_message_id, + requested_target=receipt_requested_target, + actual_target=TransportTarget( + platform="telegram", + chat_id=self._transport_identity_text( + chat_id, field="chat_id", + ), + thread_id=( + self._transport_identity_text( + effective_thread_id, field="thread_id", + ) + if effective_thread_id is not None else None + ), + ), + component="text", ordinal=i, + )) # Re-trigger typing indicator after sending a message. # Telegram clears the typing state when a new message is delivered, @@ -5778,14 +6087,16 @@ async def send( except Exception: pass # Typing failures are non-fatal + message_id = message_ids[0] if message_ids else None return SendResult( success=True, - message_id=message_ids[0] if message_ids else None, + message_id=message_id, raw_response={ "message_ids": message_ids, "requested_thread_id": requested_thread_id, "thread_fallback": used_thread_fallback, }, + receipts=tuple(receipts), ) except Exception as e: @@ -5800,7 +6111,10 @@ async def send( "[%s] send() content too long, falling back to new-message continuation", self.name, ) - return SendResult(success=False, error="message_too_long", error_kind="too_long") + return SendResult( + success=False, error="message_too_long", error_kind="too_long", + receipts=tuple(locals().get("receipts", ())), + ) # TimedOut usually means the request may have reached Telegram — # mark as non-retryable so _send_with_retry() doesn't re-send. # Exceptions: a wrapped ConnectTimeout (no connection established) @@ -5815,6 +6129,7 @@ async def send( error=safe_error, retryable=(is_connect_timeout or is_pool_timeout or not is_timeout), error_kind=error_kind, + receipts=tuple(locals().get("receipts", ())), ) async def send_or_update_status( @@ -8018,8 +8333,10 @@ async def send_voice( if not self._bot: return SendResult(success=False, error="Not connected") + receipt_plan: Optional[tuple[TransportTarget, int]] = None _transcoded_voice_path: Optional[str] = None try: + metadata = self._validated_transport_metadata(chat_id, metadata) if not os.path.exists(audio_path): return SendResult(success=False, error=self._missing_media_path_error("Audio", audio_path)) @@ -8075,19 +8392,25 @@ async def send_voice( else: _caption_variants.append((None, None)) + _audio_thread = self._metadata_thread_id(metadata) + reply_to_id = self._reply_to_message_id_for_send( + reply_to, metadata, reply_to_mode=self._reply_to_mode + ) + media_thread_kwargs = self._thread_kwargs_for_send( + chat_id, + _audio_thread, + metadata, + reply_to_message_id=reply_to_id, + reply_to_mode=self._reply_to_mode, + ) + receipt_plan, actual_thread = self._transport_media_receipt_context( + chat_id, metadata, media_thread_kwargs + ) + with open(audio_path, "rb") as audio_file: ext = os.path.splitext(audio_path)[1].lower() # .ogg / .opus files -> send as voice (round playable bubble) if ext in {".ogg", ".opus"}: - _voice_thread = self._metadata_thread_id(metadata) - reply_to_id = self._reply_to_message_id_for_send(reply_to, metadata, reply_to_mode=self._reply_to_mode) - voice_thread_kwargs = self._thread_kwargs_for_send( - chat_id, - _voice_thread, - metadata, - reply_to_message_id=reply_to_id, - reply_to_mode=self._reply_to_mode - ) msg = None _last_parse_error: Optional[Exception] = None for _cap_text, _cap_parse_mode in _caption_variants: @@ -8102,13 +8425,14 @@ async def send_voice( "reply_to_message_id": reply_to_id, "duration": _duration_secs, "read_timeout": _MEDIA_SEND_READ_TIMEOUT, - **voice_thread_kwargs, + **media_thread_kwargs, **self._notification_kwargs(metadata), }, metadata, reply_to_id, "voice", reset_media=lambda: audio_file.seek(0), + actual_thread_out=actual_thread, ) break except Exception as _cap_error: @@ -8134,15 +8458,6 @@ async def send_voice( ) elif ext in {".mp3", ".m4a"}: # Telegram's Bot API sendAudio only accepts MP3 / M4A. - _audio_thread = self._metadata_thread_id(metadata) - reply_to_id = self._reply_to_message_id_for_send(reply_to, metadata, reply_to_mode=self._reply_to_mode) - audio_thread_kwargs = self._thread_kwargs_for_send( - chat_id, - _audio_thread, - metadata, - reply_to_message_id=reply_to_id, - reply_to_mode=self._reply_to_mode - ) msg = await self._send_with_dm_topic_reply_anchor_retry( self._bot.send_audio, { @@ -8152,13 +8467,14 @@ async def send_voice( "reply_to_message_id": reply_to_id, "duration": _duration_secs, "read_timeout": _MEDIA_SEND_READ_TIMEOUT, - **audio_thread_kwargs, + **media_thread_kwargs, **self._notification_kwargs(metadata), }, metadata, reply_to_id, "audio", reset_media=lambda: audio_file.seek(0), + actual_thread_out=actual_thread, ) else: # Formats Telegram can't play natively (.wav, .flac, ...) @@ -8170,8 +8486,20 @@ async def send_voice( reply_to=reply_to, metadata=metadata, ) - return SendResult(success=True, message_id=str(msg.message_id)) + return self._transport_media_send_result( + msg, chat_id, receipt_plan, actual_thread + ) + except _InvalidTransportReceiptMetadata: + return self._invalid_transport_media_receipt_result() except Exception as e: + if receipt_plan is not None: + logger.warning( + "[%s] Telegram voice/audio outcome is unknown; " + "suppressing fallback: %s", + self.name, + _redact_telegram_error_text(e), + ) + return self._transport_media_unknown_result(receipt_plan) logger.error( "[%s] Failed to send Telegram voice/audio, falling back to base adapter: %s", self.name, @@ -8335,7 +8663,9 @@ async def send_image_file( if not self._bot: return SendResult(success=False, error="Not connected") + receipt_plan: Optional[tuple[TransportTarget, int]] = None try: + metadata = self._validated_transport_metadata(chat_id, metadata) if not os.path.exists(image_path): return SendResult(success=False, error=self._missing_media_path_error("Image", image_path)) @@ -8348,6 +8678,9 @@ async def send_image_file( reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode ) + receipt_plan, actual_thread = self._transport_media_receipt_context( + chat_id, metadata, thread_kwargs + ) with open(image_path, "rb") as image_file: msg = await self._send_with_dm_topic_reply_anchor_retry( self._bot.send_photo, @@ -8364,39 +8697,51 @@ async def send_image_file( reply_to_id, "photo", reset_media=lambda: image_file.seek(0), + actual_thread_out=actual_thread, ) - return SendResult(success=True, message_id=str(msg.message_id)) + return self._transport_media_send_result( + msg, chat_id, receipt_plan, actual_thread + ) + except _InvalidTransportReceiptMetadata: + return self._invalid_transport_media_receipt_result() except Exception as e: - error_str = str(e) - # Dimension-related errors are the expected case for valid image - # files that Telegram just refuses as photos (screenshots, extreme - # aspect ratios). Log at INFO because the document fallback is - # the correct path. Any other send_photo failure also falls back - # to document (rate limits, corrupt file markers, format edge - # cases), but at WARNING because it's unexpected and worth - # surfacing in logs. - is_dim_error = ( - "Photo_invalid_dimensions" in error_str - or "PHOTO_INVALID_DIMENSIONS" in error_str - ) - if is_dim_error: - logger.info( - "[%s] Image dimensions exceed Telegram photo limits, " - "sending as document: %s", - self.name, - image_path, - ) + # A document fallback is safe only for Telegram's exact, typed + # pre-dispatch rejection. Substring matching is unsafe: an + # ambiguous timeout may include the marker after the photo was + # already accepted and would then cause a duplicate send. + try: + from telegram.error import BadRequest + except ImportError: + is_dim_error = False else: + is_dim_error = ( + isinstance(e, BadRequest) + and str(e).strip().casefold() == "photo_invalid_dimensions" + ) + if not is_dim_error: logger.warning( - "[%s] Failed to send Telegram local image as photo, " - "trying document fallback: %s", + "[%s] Telegram photo outcome is unknown; " + "suppressing fallback: %s", self.name, _redact_telegram_error_text(e), - exc_info=True, ) - # Fallback to sending as document (file) — no dimension limit, - # only 50MB size limit. If even that fails, fall back to the - # base adapter's text-only "Image: /path" rendering. + if receipt_plan is not None: + return self._transport_media_unknown_result(receipt_plan) + return SendResult( + success=False, + error="Telegram media delivery outcome is unknown", + error_kind="unknown", + retryable=False, + ) + logger.info( + "[%s] Image dimensions exceed Telegram photo limits, " + "sending as document: %s", + self.name, + image_path, + ) + # The exact PHOTO_INVALID_DIMENSIONS BadRequest is a definite + # pre-dispatch rejection, so sending the same component as a + # document cannot duplicate an accepted photo. try: return await self.send_document( chat_id=chat_id, @@ -8430,7 +8775,9 @@ async def send_document( if not self._bot: return SendResult(success=False, error="Not connected") + receipt_plan: Optional[tuple[TransportTarget, int]] = None try: + metadata = self._validated_transport_metadata(chat_id, metadata) if not os.path.exists(file_path): return SendResult(success=False, error=self._missing_media_path_error("File", file_path)) @@ -8445,6 +8792,9 @@ async def send_document( reply_to_mode=self._reply_to_mode ) + receipt_plan, actual_thread = self._transport_media_receipt_context( + chat_id, metadata, thread_kwargs + ) with open(file_path, "rb") as f: msg = await self._send_with_dm_topic_reply_anchor_retry( self._bot.send_document, @@ -8462,9 +8812,22 @@ async def send_document( reply_to_id, "document", reset_media=lambda: f.seek(0), + actual_thread_out=actual_thread, ) - return SendResult(success=True, message_id=str(msg.message_id)) + return self._transport_media_send_result( + msg, chat_id, receipt_plan, actual_thread + ) + except _InvalidTransportReceiptMetadata: + return self._invalid_transport_media_receipt_result() except Exception as e: + if receipt_plan is not None: + logger.warning( + "[%s] Telegram document outcome is unknown; " + "suppressing fallback: %s", + self.name, + _redact_telegram_error_text(e), + ) + return self._transport_media_unknown_result(receipt_plan) logger.warning( "[%s] Failed to send document: %s", self.name, _redact_telegram_error_text(e), @@ -8484,7 +8847,9 @@ async def send_video( if not self._bot: return SendResult(success=False, error="Not connected") + receipt_plan: Optional[tuple[TransportTarget, int]] = None try: + metadata = self._validated_transport_metadata(chat_id, metadata) if not os.path.exists(video_path): return SendResult(success=False, error=self._missing_media_path_error("Video", video_path)) @@ -8497,6 +8862,9 @@ async def send_video( reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode ) + receipt_plan, actual_thread = self._transport_media_receipt_context( + chat_id, metadata, thread_kwargs + ) with open(video_path, "rb") as f: msg = await self._send_with_dm_topic_reply_anchor_retry( self._bot.send_video, @@ -8513,9 +8881,22 @@ async def send_video( reply_to_id, "video", reset_media=lambda: f.seek(0), + actual_thread_out=actual_thread, ) - return SendResult(success=True, message_id=str(msg.message_id)) + return self._transport_media_send_result( + msg, chat_id, receipt_plan, actual_thread + ) + except _InvalidTransportReceiptMetadata: + return self._invalid_transport_media_receipt_result() except Exception as e: + if receipt_plan is not None: + logger.warning( + "[%s] Telegram video outcome is unknown; " + "suppressing fallback: %s", + self.name, + _redact_telegram_error_text(e), + ) + return self._transport_media_unknown_result(receipt_plan) logger.warning( "[%s] Failed to send video: %s", self.name, _redact_telegram_error_text(e), diff --git a/tests/cron/test_cron_bot_chat_delivery.py b/tests/cron/test_cron_bot_chat_delivery.py index 92ebadde4903..2b3846376cc4 100644 --- a/tests/cron/test_cron_bot_chat_delivery.py +++ b/tests/cron/test_cron_bot_chat_delivery.py @@ -15,6 +15,7 @@ from cron.scheduler import ( BOT_CHAT_PLATFORM, _deliver_to_bot_chat, + _normalize_deliver_value, _preflight_check_delivery, _resolve_bot_chat_target, _resolve_delivery_targets, @@ -24,6 +25,41 @@ # ── token parsing ──────────────────────────────────────────────────────────── +def test_deliver_normalization_rejects_private_non_string_values_without_stringifying(): + class PrivateValue: + def __str__(self): + raise AssertionError("private value must not be stringified") + + private = PrivateValue() + assert _normalize_deliver_value(private) == "local" + assert _normalize_deliver_value(["telegram", private, " origin "]) == ( + "telegram,origin" + ) + assert _normalize_deliver_value({"target": "private"}) == "local" + + +def test_deliver_normalization_rejects_builtin_subclasses_before_magic_methods(): + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile text truthiness was evaluated") + + def strip(self, *_args, **_kwargs): + raise AssertionError("hostile text strip was called") + + class HostileList(list): + def __iter__(self): + raise AssertionError("hostile list was iterated") + + class HostileTuple(tuple): + def __iter__(self): + raise AssertionError("hostile tuple was iterated") + + assert _normalize_deliver_value(HostileText("telegram")) == "local" + assert _normalize_deliver_value(HostileList(["telegram"])) == "local" + assert _normalize_deliver_value(HostileTuple(("telegram",))) == "local" + assert _normalize_deliver_value([HostileText("telegram"), "origin"]) == "origin" + + def test_bare_token_targets_own_profile(): assert parse_bot_chat_deliver_token("bot-chat") == "" assert parse_bot_chat_deliver_token(" Bot-Chat ") == "" @@ -47,7 +83,9 @@ def test_non_bot_chat_tokens_pass_through(): def test_own_profile_resolves_without_name(): target = _resolve_bot_chat_target({"id": "j1"}, "") - assert target == {"platform": BOT_CHAT_PLATFORM, "chat_id": "", "thread_id": None} + assert target == { + "platform": BOT_CHAT_PLATFORM, "chat_id": "_self", "thread_id": None, + } def test_named_profile_resolves_when_exists(): @@ -169,8 +207,20 @@ def test_deliver_failure_returns_error_string(): sched.subprocess, "run", return_value=_completed(returncode=1, stderr="boom") ), mock.patch.object(sched.shutil, "which", return_value="/usr/bin/hermes"): err = _deliver_to_bot_chat({"id": "j1", "name": "n"}, "out", "") - assert err is not None - assert "boom" in err + assert err == "bot-chat delivery confirmation unavailable" + assert "boom" not in err + + +def test_deliver_exception_is_categorical_without_stringifying_private_error(): + class PrivateError(Exception): + def __str__(self): + raise AssertionError("private exception must not be stringified") + + with mock.patch.object( + sched.subprocess, "run", side_effect=PrivateError() + ), mock.patch.object(sched.shutil, "which", return_value="/usr/bin/hermes"): + err = _deliver_to_bot_chat({"id": "j1", "name": "n"}, "out", "") + assert err == "bot-chat delivery confirmation unavailable" def test_deliver_timeout_returns_error_string(): @@ -179,8 +229,7 @@ def test_deliver_timeout_returns_error_string(): side_effect=subprocess.TimeoutExpired(cmd="hermes", timeout=600), ), mock.patch.object(sched.shutil, "which", return_value="/usr/bin/hermes"): err = _deliver_to_bot_chat({"id": "j1", "name": "n"}, "out", "") - assert err is not None - assert "timed out" in err + assert err == "bot-chat delivery confirmation unavailable" def test_deliver_message_carries_cron_attribution(tmp_path): diff --git a/tests/cron/test_cron_created_delivery.py b/tests/cron/test_cron_created_delivery.py index 8a71abae799c..60b6f7500c4e 100644 --- a/tests/cron/test_cron_created_delivery.py +++ b/tests/cron/test_cron_created_delivery.py @@ -72,6 +72,13 @@ def _create(deliver=None): ) +def _stored_deliver(result): + from cron.jobs import get_job + + assert "deliver" not in result + return get_job(result["job_id"])["deliver"] + + class TestCronContextDeliveryResolution: def test_omitted_deliver_resolves_to_creator_target(self, temp_cron_home): tokens, extra = _enter_cron_context("telegram", "-100123456", "17") @@ -80,7 +87,7 @@ def test_omitted_deliver_resolves_to_creator_target(self, temp_cron_home): finally: _exit_cron_context(tokens, extra) assert result["success"] is True - assert result["deliver"] == "telegram:-100123456:17" + assert _stored_deliver(result) == "telegram:-100123456:17" def test_literal_origin_resolves_to_creator_target(self, temp_cron_home): tokens, extra = _enter_cron_context("telegram", "-100123456", "17") @@ -88,7 +95,7 @@ def test_literal_origin_resolves_to_creator_target(self, temp_cron_home): result = _create(deliver="origin") finally: _exit_cron_context(tokens, extra) - assert result["deliver"] == "telegram:-100123456:17" + assert _stored_deliver(result) == "telegram:-100123456:17" def test_no_thread_id_omits_thread_segment(self, temp_cron_home): tokens, extra = _enter_cron_context("slack", "C0ABC") @@ -96,7 +103,7 @@ def test_no_thread_id_omits_thread_segment(self, temp_cron_home): result = _create(deliver="origin") finally: _exit_cron_context(tokens, extra) - assert result["deliver"] == "slack:C0ABC" + assert _stored_deliver(result) == "slack:C0ABC" def test_creator_without_delivery_target_falls_back_to_local(self, temp_cron_home): # Creator job delivers nowhere concrete (deliver='local' run): @@ -106,7 +113,7 @@ def test_creator_without_delivery_target_falls_back_to_local(self, temp_cron_hom result = _create(deliver="origin") finally: _exit_cron_context(tokens, extra) - assert result["deliver"] == "local" + assert _stored_deliver(result) == "local" def test_comma_list_resolves_only_origin_element(self, temp_cron_home): tokens, extra = _enter_cron_context("telegram", "-100123456", "17") @@ -114,7 +121,7 @@ def test_comma_list_resolves_only_origin_element(self, temp_cron_home): result = _create(deliver="origin,all") finally: _exit_cron_context(tokens, extra) - assert result["deliver"] == "telegram:-100123456:17,all" + assert _stored_deliver(result) == "telegram:-100123456:17,all" def test_explicit_target_passes_through_verbatim(self, temp_cron_home): tokens, extra = _enter_cron_context("telegram", "-100999", "3") @@ -122,7 +129,7 @@ def test_explicit_target_passes_through_verbatim(self, temp_cron_home): result = _create(deliver="discord:#engineering") finally: _exit_cron_context(tokens, extra) - assert result["deliver"] == "discord:#engineering" + assert _stored_deliver(result) == "discord:#engineering" def test_local_passes_through(self, temp_cron_home): tokens, extra = _enter_cron_context("telegram", "-100999") @@ -130,7 +137,7 @@ def test_local_passes_through(self, temp_cron_home): result = _create(deliver="local") finally: _exit_cron_context(tokens, extra) - assert result["deliver"] == "local" + assert _stored_deliver(result) == "local" def test_stored_deliver_never_literal_origin_in_cron_context(self, temp_cron_home): from cron.jobs import get_job @@ -184,7 +191,7 @@ def test_chat_session_create_keeps_literal_origin(self, temp_cron_home): # No cron_session var — ordinary chat/CLI create. Existing semantics: # 'origin' stays literal and resolves at fire time. result = _create(deliver="origin") - assert result["deliver"] == "origin" + assert _stored_deliver(result) == "origin" def test_chat_session_omitted_deliver_unchanged(self, temp_cron_home): # Outside cron context the resolution helper must be a no-op so the diff --git a/tests/cron/test_cron_failure_deliver.py b/tests/cron/test_cron_failure_deliver.py index 88e6fee2441f..d32b1bbe6548 100644 --- a/tests/cron/test_cron_failure_deliver.py +++ b/tests/cron/test_cron_failure_deliver.py @@ -17,6 +17,7 @@ import cron.scheduler as s from cron.scheduler import _resolve_delivery_targets +from gateway.platforms.base import TransportReceipt, TransportTarget @pytest.fixture @@ -58,7 +59,21 @@ def run_env(monkeypatch, tmp_path): async def fake_sender(pconfig, chat_id, message, *, thread_id=None, media_files=None, force_document=False, caption=None): send_calls.append({"chat_id": chat_id, "message": message}) - return {"success": True, "chat_id": chat_id, "message_id": "1.2"} + target = TransportTarget( + "slack", str(chat_id), str(thread_id) if thread_id is not None else None, + ) + receipt = TransportReceipt( + outcome="delivered", + requested_target=target, + actual_target=target, + provider_message_id="1.2", + ) + return { + "success": True, + "chat_id": chat_id, + "message_id": "1.2", + "receipts": (receipt,), + } import gateway.platform_registry as reg import hermes_cli.plugins as hp @@ -77,6 +92,22 @@ async def fake_sender(pconfig, chat_id, message, *, thread_id=None, monkeypatch.setattr(s, "create_execution", lambda *_a, **_kw: {"id": "exec-t"}) monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) + monkeypatch.setattr( + s, + "preregister_receipt_plan", + lambda _execution_id, *, fire_identity, components: [ + { + "id": f"attempt-{index}", + "platform": component["target"]["platform"], + "chat_id": component["target"]["chat_id"], + "thread_id": component["target"]["thread_id"], + "component": component["component"], + "ordinal": component["ordinal"], + } + for index, component in enumerate(components) + ], + ) + monkeypatch.setattr(s, "record_transport_receipt", lambda *_a, **_kw: True) monkeypatch.setattr( s, "save_job_output", lambda jid, out: state["saved"].append(jid) or f"/tmp/{jid}.txt", diff --git a/tests/cron/test_cron_live_delivery_confirmation.py b/tests/cron/test_cron_live_delivery_confirmation.py index b2822edc2402..a65986272af0 100644 --- a/tests/cron/test_cron_live_delivery_confirmation.py +++ b/tests/cron/test_cron_live_delivery_confirmation.py @@ -26,23 +26,22 @@ from cron import scheduler as sched from cron.scheduler import _confirm_adapter_delivery, _deliver_result from gateway.config import Platform, PlatformConfig +from gateway.platforms.base import SendResult, TransportReceipt, TransportTarget + + +def _SendResult(success=True, message_id=None, raw_response=None): + """Build the real adapter result type while keeping concise test defaults.""" + return SendResult( + success=success, + message_id=message_id, + raw_response=raw_response, + ) # --------------------------------------------------------------------------- # _confirm_adapter_delivery: the contract in isolation # --------------------------------------------------------------------------- -class _SendResult: - """Minimal stand-in for an adapter SendResult.""" - - def __init__(self, success=True, message_id=None, raw_response=None, **extra): - self.success = success - self.message_id = message_id - self.raw_response = raw_response - for key, value in extra.items(): - setattr(self, key, value) - - class TestConfirmAdapterDelivery: def test_none_is_not_delivered(self): assert _confirm_adapter_delivery(None, "j1") is False @@ -60,8 +59,12 @@ def test_filtered_dict_is_not_delivered(self): filtered = {"success": True, "filtered": "silence_narration", "delivered": False} assert _confirm_adapter_delivery(filtered, "j1") is False - def test_delivered_false_on_an_object_is_not_delivered(self): - result = _SendResult(success=True, message_id=42, delivered=False) + def test_duck_typed_object_is_not_delivered(self): + class DuckResult: + success = True + message_id = 42 + + result = DuckResult() assert _confirm_adapter_delivery(result, "j1") is False def test_positive_evidence_is_delivered_without_warning(self, caplog): @@ -138,6 +141,26 @@ def _run(job, content, send_result, relay=False, standalone_result=None, cron_cf Returns ``(error, router_calls, standalone_calls)``. ``cron_cfg`` extends the ``cron:`` section handed to the scheduler (default: unwrapped output). """ + if ( + type(send_result) is SendResult + and send_result.message_id is not None + and not send_result.receipts + ): + origin = job["origin"] + target = TransportTarget( + origin["platform"], + str(origin["chat_id"]), + str(origin["thread_id"]) if origin.get("thread_id") is not None else None, + ) + send_result.receipts = ( + TransportReceipt( + outcome="delivered", + requested_target=target, + actual_target=target, + provider_message_id=str(send_result.message_id), + ), + ) + loop = MagicMock() loop.is_running.return_value = True @@ -181,11 +204,12 @@ class TestFilteredResultIsNotDelivered: def test_filtered_dict_does_not_log_a_live_delivery(self, caplog): with caplog.at_level(logging.INFO, logger="cron.scheduler"): - _, router_calls, standalone_calls = _run(_job(), "...", self.FILTERED) + error, router_calls, standalone_calls = _run(_job(), "...", self.FILTERED) assert len(router_calls) == 1 # the live send was attempted assert "via live adapter" not in caplog.text # but never claimed as delivered - assert len(standalone_calls) == 1 # fell back instead of lying + assert standalone_calls == [] # no blind retry after dispatch + assert error is not None and "unconfirmed result" in error def test_filtered_dict_fails_closed_on_the_relay_lane(self): """Relay owns the destination, so there is no fallback — report it.""" @@ -253,13 +277,13 @@ def test_log_includes_thread_and_message_id(self, caplog): assert error is None assert "via live adapter thread=99 message_id=1234" in caplog.text - def test_log_uses_a_dash_when_the_lane_is_unknown(self, caplog): - """No thread and an evidence-free result must still be attributable.""" + def test_evidence_free_result_is_not_logged_as_delivered(self, caplog): + """No receipt means unknown even when the legacy success bit is true.""" with caplog.at_level(logging.INFO, logger="cron.scheduler"): error, _, _ = _run(_job(), "Nightly report.", _SendResult()) - assert error is None - assert "via live adapter thread=- message_id=-" in caplog.text + assert error is not None and "typed receipt" in error + assert "via live adapter" not in caplog.text assert "UNVERIFIED" in caplog.text @@ -294,8 +318,25 @@ def test_media_route_metadata_carries_notify(self, tmp_path): media.write_bytes(b"\x89PNG\r\n\x1a\n") sent = [] - def fake_send_media(adapter, chat_id, media_files, metadata, loop, job, platform=None): + def fake_send_media( + adapter, chat_id, media_files, metadata, loop, job, + platform=None, receipts_out=None, + ): sent.append({"media": list(media_files), "metadata": metadata}) + requested = metadata["_transport_receipt_requested_target"] + target = TransportTarget( + requested["platform"], requested["chat_id"], + requested.get("thread_id") or None, + ) + assert receipts_out is not None + receipts_out.append(TransportReceipt( + outcome="delivered", + requested_target=target, + actual_target=target, + provider_message_id="media-1", + component="media", + ordinal=0, + )) return [] with patch("cron.scheduler._send_media_via_adapter", side_effect=fake_send_media), \ @@ -337,8 +378,25 @@ def test_explicit_false_disables_notify_on_media_route(self, tmp_path): media.write_bytes(b"\x89PNG\r\n\x1a\n") sent = [] - def fake_send_media(adapter, chat_id, media_files, metadata, loop, job, platform=None): + def fake_send_media( + adapter, chat_id, media_files, metadata, loop, job, + platform=None, receipts_out=None, + ): sent.append(metadata) + requested = metadata["_transport_receipt_requested_target"] + target = TransportTarget( + requested["platform"], requested["chat_id"], + requested.get("thread_id") or None, + ) + assert receipts_out is not None + receipts_out.append(TransportReceipt( + outcome="delivered", + requested_target=target, + actual_target=target, + provider_message_id="media-1", + component="media", + ordinal=0, + )) return [] with patch("cron.scheduler._send_media_via_adapter", side_effect=fake_send_media), \ @@ -366,13 +424,11 @@ def test_default_config_ships_notify_true(self): class TestUnverifiedDeliveryIsRecordedOnTheJob: - """An evidence-free ack is accepted, but the state must reach the job - record (and from there ``hermes cron list`` / ``cron doctor``), not only a - WARNING log line.""" + """An evidence-free operation ack stays unknown and remains visible.""" def test_evidence_free_ack_records_the_target(self): error, _, _ = _run(_job(), "Nightly report.", _SendResult()) - assert error is None + assert error is not None and "typed receipt" in error assert RECORDED_VERIFICATION == [("92e639af907f", [f"telegram:{CHAT_ID}"])] def test_positive_evidence_clears_the_marker(self): @@ -395,8 +451,12 @@ def test_recorder_clears_a_stale_marker(self): def test_tool_listing_exposes_the_field(self): from tools.cronjob_tools import _format_job - assert _format_job({"id": "j1", "name": "n", "prompt": "p", - "last_delivery_unverified": ["slack:C1"]})["last_delivery_unverified"] == ["slack:C1"] + formatted = _format_job({ + "id": "j1", "name": "n", "prompt": "p", + "last_delivery_unverified": ["slack:C1"], + }) + assert formatted["last_delivery_unverified"] is True + assert "slack:C1" not in str(formatted) def test_scheduler_module_exposes_the_confirmation_helper(): diff --git a/tests/cron/test_cron_reasoning_effort.py b/tests/cron/test_cron_reasoning_effort.py index 4f45549a0505..3c58beb48750 100644 --- a/tests/cron/test_cron_reasoning_effort.py +++ b/tests/cron/test_cron_reasoning_effort.py @@ -168,19 +168,22 @@ def test_job_effort_is_model_independent(self): class TestCronjobToolReasoningEffort: - """The model tool READS the field (list surfacing) but must never WRITE - it: models don't make model-configuration decisions (standing policy — - the only exception is user-defined profile selection). The pin is set - via `hermes cron create/edit --reasoning-effort` only.""" + """The model tool must neither reveal nor write the field. - def test_format_job_surfaces_pin_when_set(self, tmp_cron_dir): + Models don't make model-configuration decisions (standing policy — the + only exception is user-defined profile selection). The pin is set via + `hermes cron create/edit --reasoning-effort` only. + """ + + def test_format_job_redacts_pin_when_set(self, tmp_cron_dir): import json from tools.cronjob_tools import cronjob _create(reasoning_effort="high") + assert load_jobs()[0]["reasoning_effort"] == "high" listed = json.loads(cronjob(action="list"))["jobs"][0] - assert listed["reasoning_effort"] == "high" + assert "reasoning_effort" not in listed def test_format_job_omits_field_when_unset(self, tmp_cron_dir): import json diff --git a/tests/cron/test_cron_script.py b/tests/cron/test_cron_script.py index 5c51134d25bb..1d76fc99274f 100644 --- a/tests/cron/test_cron_script.py +++ b/tests/cron/test_cron_script.py @@ -448,6 +448,7 @@ class TestCronjobToolScript: def test_clear_script(self, cron_env, monkeypatch): monkeypatch.setenv("HERMES_INTERACTIVE", "1") + from cron.jobs import get_job from tools.cronjob_tools import cronjob create_result = json.loads(cronjob( @@ -465,22 +466,25 @@ def test_clear_script(self, cron_env, monkeypatch): )) assert update_result["success"] is True assert "script" not in update_result["job"] + assert get_job(job_id)["script"] is None - def test_list_shows_script(self, cron_env, monkeypatch): + def test_list_redacts_script_but_store_preserves_it(self, cron_env, monkeypatch): monkeypatch.setenv("HERMES_INTERACTIVE", "1") + from cron.jobs import get_job from tools.cronjob_tools import cronjob - cronjob( + create_result = json.loads(cronjob( action="create", schedule="every 1h", prompt="Monitor things", script="data_collector.py", - ) + )) + assert get_job(create_result["job_id"])["script"] == "data_collector.py" list_result = json.loads(cronjob(action="list")) assert list_result["success"] is True assert len(list_result["jobs"]) == 1 - assert list_result["jobs"][0]["script"] == "data_collector.py" + assert "script" not in list_result["jobs"][0] class TestScriptPathContainment: diff --git a/tests/cron/test_execution_ledger.py b/tests/cron/test_execution_ledger.py index d68259ae4b4f..03b0ada4378c 100644 --- a/tests/cron/test_execution_ledger.py +++ b/tests/cron/test_execution_ledger.py @@ -7,6 +7,7 @@ import sqlite3 import subprocess import sys +from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -17,6 +18,84 @@ def _point_ledger(monkeypatch, tmp_path): return executions +def test_scheduled_fire_identity_rejects_private_job_id_without_stringifying(): + import cron.executions as executions + + class PrivateJobId: + def __str__(self): + raise AssertionError("private job id was stringified") + + def __repr__(self): + raise AssertionError("private job id was represented") + + with __import__("pytest").raises(ValueError, match="job_id must be a string"): + executions.scheduled_fire_identity( + PrivateJobId(), "2026-08-23T20:00:00+00:00" + ) + + +def test_create_execution_rejects_private_identity_without_stringifying( + monkeypatch, tmp_path +): + executions = _point_ledger(monkeypatch, tmp_path) + + class PrivateIdentity: + def __str__(self): + raise AssertionError("private identity was stringified") + + def __repr__(self): + raise AssertionError("private identity was represented") + + with __import__("pytest").raises(ValueError, match="job_id must be a string"): + executions.create_execution(PrivateIdentity(), source="direct") + with __import__("pytest").raises(ValueError, match="source must be a string"): + executions.create_execution("safe-job", source=PrivateIdentity()) + + +def test_list_executions_rejects_private_filters_without_stringifying( + monkeypatch, tmp_path +): + executions = _point_ledger(monkeypatch, tmp_path) + + class PrivateFilter: + def __str__(self): + raise AssertionError("private filter was stringified") + + def __repr__(self): + raise AssertionError("private filter was represented") + + with __import__("pytest").raises(ValueError, match="job_id must be a string"): + executions.list_executions(job_id=PrivateFilter()) + with __import__("pytest").raises( + ValueError, match="before_claimed_at must be a string" + ): + executions.list_executions(before_claimed_at=PrivateFilter()) + with __import__("pytest").raises(ValueError, match="limit must be an integer"): + executions.list_executions(limit=True) + + +def test_latest_executions_rejects_private_ids_before_hashing(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + + class PrivateJobId: + def __hash__(self): + raise AssertionError("private job id was hashed") + + def __eq__(self, _other): + raise AssertionError("private job id was compared") + + def __str__(self): + raise AssertionError("private job id was stringified") + + def __repr__(self): + raise AssertionError("private job id was represented") + + with __import__("pytest").raises(ValueError, match="job_id must be a string"): + executions.latest_executions([PrivateJobId()]) + with __import__("pytest").raises(ValueError, match="job_ids must be a list"): + executions.latest_executions(("safe",)) + + def test_execution_transitions_are_durable(monkeypatch, tmp_path): executions = _point_ledger(monkeypatch, tmp_path) @@ -39,6 +118,653 @@ def test_execution_transitions_are_durable(monkeypatch, tmp_path): assert persisted == [completed] +def test_execution_claim_persists_fire_identity_before_dispatch(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + + manual = executions.create_execution("manual-job", source="manual") + assert manual["fire_identity"] == manual["id"] + + scheduled = executions.create_execution( + "scheduled-job", source="builtin", fire_identity="scheduled-fire-2026-08-22T19:00Z", + ) + assert scheduled["fire_identity"] == "scheduled-fire-2026-08-22T19:00Z" + + +def test_execution_fire_identity_binds_once_before_receipt_plan(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("bind-job", source="external") + + bound = executions.bind_execution_fire_identity(execution["id"], "claim-fire") + assert bound["fire_identity"] == "claim-fire" + with __import__("pytest").raises(ValueError, match="already bound"): + executions.bind_execution_fire_identity(execution["id"], "different-fire") + + +def test_new_execution_without_transport_plan_is_not_attempted_unconfirmed(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + local = executions.create_execution("local-only", source="builtin") + executions.finish_execution(local["id"], success=True) + + assert executions.receipt_summary(local["id"]) == { + "delivered": 0, "failed": 0, "unknown": 0, "targets_delivered": 0, + } + + +def test_receipt_plan_preregisters_unknown_attempt_without_persisting_content(monkeypatch, tmp_path): + """Dispatch may begin only after every content-free receipt row is durable.""" + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("job-receipt", source="builtin") + + attempts = executions.preregister_receipt_plan( + execution["id"], + fire_identity="fire-1", + components=[ + { + "target": {"platform": "matrix", "chat_id": "!room:example.org"}, + "component": "text", + "ordinal": 0, + "content": "do not persist this report body", + }, + { + "target": {"platform": "matrix", "chat_id": "!room:example.org"}, + "component": "media", + "ordinal": 1, + "content": "logical media item identity", + }, + ], + ) + + assert len(attempts) == 2 + assert {row["outcome"] for row in attempts} == {"unknown"} + summary = executions.receipt_summary(execution["id"]) + assert summary == {"delivered": 0, "failed": 0, "unknown": 2, "targets_delivered": 0} + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + dumped = "\n".join(str(row) for row in conn.execute( + "SELECT content_hash FROM delivery_components" + )) + assert "do not persist this report body" not in dumped + assert "logical media item identity" not in dumped + + +def test_receipt_plan_rejects_private_non_string_content_without_stringifying( + monkeypatch, tmp_path +): + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("hostile-content", source="direct") + + class PrivateContent: + def __str__(self): + raise AssertionError("private content was stringified") + + def __repr__(self): + raise AssertionError("private content was represented") + + with __import__("pytest").raises(ValueError, match="content must be a string"): + executions.preregister_receipt_plan( + execution["id"], + fire_identity=execution["fire_identity"], + components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": PrivateContent(), + }], + ) + assert executions.receipt_summary(execution["id"])["unknown"] == 0 + + +def test_receipt_plan_rejects_container_and_text_subclasses_before_magic_methods( + monkeypatch, tmp_path +): + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("hostile-plan", source="direct") + + class HostileList(list): + def __iter__(self): + raise AssertionError("hostile list was iterated") + + def __len__(self): + raise AssertionError("hostile list length was evaluated") + + class HostileDict(dict): + def get(self, *_args, **_kwargs): + raise AssertionError("hostile mapping get was called") + + def __getitem__(self, _key): + raise AssertionError("hostile mapping item access was called") + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile text truthiness was evaluated") + + def __len__(self): + raise AssertionError("hostile text length was evaluated") + + def __eq__(self, _other): + raise AssertionError("hostile text was compared") + + def __hash__(self): + raise AssertionError("hostile text was hashed") + + valid = { + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "safe", + } + invalid_plans = ( + HostileList([valid]), + [HostileDict(valid)], + [{**valid, "target": HostileDict(valid["target"])}], + [{**valid, "content": HostileText("safe")}], + ) + for invalid in invalid_plans: + with __import__("pytest").raises(ValueError): + executions.preregister_receipt_plan( + execution["id"], + fire_identity=execution["fire_identity"], + components=invalid, + ) + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + assert conn.execute("SELECT COUNT(*) FROM delivery_targets").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM delivery_components").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 0 + + +def test_receipt_transition_only_records_typed_ack_and_target_is_all_components(monkeypatch, tmp_path): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("job-ack", source="builtin") + attempts = executions.preregister_receipt_plan( + execution["id"], + fire_identity="fire-2", + components=[ + { + "target": {"platform": "telegram", "chat_id": "123", "thread_id": "8"}, + "component": "text", + "ordinal": 0, + "content": "text component", + }, + { + "target": {"platform": "telegram", "chat_id": "123", "thread_id": "8"}, + "component": "media", + "ordinal": 1, + "content": "media component", + }, + ], + ) + receipt = TransportReceipt( + outcome="delivered", + provider_message_id="42", + requested_target=TransportTarget("telegram", "123", "8"), + actual_target=TransportTarget("telegram", "123", None), + ) + + assert executions.record_transport_receipt(attempts[0]["id"], receipt) is True + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, + "failed": 0, + "unknown": 1, + "targets_delivered": 0, + } + assert executions.record_transport_receipt(attempts[0]["id"], receipt) is False + + +def test_requested_target_is_not_delivered_when_ack_landed_at_fallback_target( + monkeypatch, tmp_path +): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("fallback-target", source="builtin") + attempts = executions.preregister_receipt_plan( + execution["id"], + fire_identity="fallback-fire", + components=[{ + "target": {"platform": "telegram", "chat_id": "123", "thread_id": "8"}, + "component": "text", + "ordinal": 0, + "content": "fallback-bound text", + }], + ) + receipt = TransportReceipt( + outcome="delivered", + provider_message_id="fallback-provider-id", + requested_target=TransportTarget("telegram", "123", "8"), + actual_target=TransportTarget("telegram", "123", None), + component="text", + ordinal=0, + ) + + assert executions.record_transport_receipt(attempts[0]["id"], receipt) is True + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, + "failed": 0, + "unknown": 0, + "targets_delivered": 0, + } + + +def test_requested_target_is_delivered_when_every_ack_matches_target( + monkeypatch, tmp_path +): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("exact-target", source="builtin") + attempts = executions.preregister_receipt_plan( + execution["id"], + fire_identity="exact-fire", + components=[{ + "target": {"platform": "telegram", "chat_id": "123", "thread_id": "8"}, + "component": "text", + "ordinal": 0, + "content": "exact-target text", + }], + ) + target = TransportTarget("telegram", "123", "8") + receipt = TransportReceipt( + outcome="delivered", + provider_message_id="exact-provider-id", + requested_target=target, + actual_target=target, + component="text", + ordinal=0, + ) + + assert executions.record_transport_receipt(attempts[0]["id"], receipt) is True + assert executions.receipt_summary(execution["id"])["targets_delivered"] == 1 + + +def test_unknown_observation_requires_exact_binding_without_upgrading_outcome( + monkeypatch, tmp_path +): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("observe-unknown", source="direct") + attempts = executions.preregister_receipt_plan( + execution["id"], + fire_identity=execution["fire_identity"], + components=[{ + "target": {"platform": "bot-chat", "chat_id": "research"}, + "component": "text", "ordinal": 0, "content": "planned query", + }], + ) + wrong = TransportReceipt( + outcome="unknown", + requested_target=TransportTarget("bot-chat", "other-profile"), + ) + with __import__("pytest").raises(ValueError, match="does not match"): + executions.observe_transport_unknown(attempts[0]["id"], wrong) + + with executions._transaction() as conn: + assert conn.execute( + "SELECT observed_at FROM delivery_attempts WHERE id=?", + (attempts[0]["id"],), + ).fetchone()["observed_at"] is None + + exact = TransportReceipt( + outcome="unknown", + requested_target=TransportTarget("bot-chat", "research"), + ) + assert executions.observe_transport_unknown(attempts[0]["id"], exact) is True + assert executions.receipt_summary(execution["id"]) == { + "delivered": 0, "failed": 0, "unknown": 1, "targets_delivered": 0, + } + + +def test_receipt_plan_is_idempotent_only_for_same_fire_and_receipt_binds_target(monkeypatch, tmp_path): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("job-plan", source="builtin") + plan = [{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "secret report body", + }] + first = executions.preregister_receipt_plan(execution["id"], fire_identity="fire-a", components=plan) + assert executions.preregister_receipt_plan( + execution["id"], fire_identity="fire-a", components=plan + ) == first + with __import__("pytest").raises(ValueError, match="conflicting"): + executions.preregister_receipt_plan(execution["id"], fire_identity="fire-b", components=plan) + + wrong_target = TransportReceipt( + outcome="delivered", provider_message_id="1", + requested_target=TransportTarget("telegram", "other"), + actual_target=TransportTarget("telegram", "other"), + ) + with __import__("pytest").raises(ValueError, match="requested_target"): + executions.record_transport_receipt(first[0]["id"], wrong_target) + + independent = executions.create_execution("job-plan", source="builtin") + assert executions.preregister_receipt_plan( + independent["id"], fire_identity="fire-independent", components=plan, + ) + conflicting = executions.create_execution("job-plan", source="recovery") + changed_plan = [dict(plan[0], content="different content")] + with __import__("pytest").raises(ValueError, match="conflicting.*fire_identity"): + executions.preregister_receipt_plan( + conflicting["id"], fire_identity="fire-a", components=changed_plan, + ) + + +def test_mutated_duck_typed_actual_target_cannot_mark_attempt_delivered(monkeypatch, tmp_path): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("typed-boundary", source="builtin") + attempts = executions.preregister_receipt_plan( + execution["id"], fire_identity="typed-fire", components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + ) + target = TransportTarget("telegram", "123") + receipt = TransportReceipt( + outcome="delivered", provider_message_id="provider-1", + requested_target=target, actual_target=target, + ) + object.__setattr__(receipt, "actual_target", object()) + with __import__("pytest").raises(TypeError, match="TransportTarget"): + executions.record_transport_receipt(attempts[0]["id"], receipt) + assert executions.receipt_summary(execution["id"])["unknown"] == 1 + + +def test_receipt_persistence_rejects_subclasses_before_magic_methods( + monkeypatch, tmp_path +): + from datetime import datetime, timezone + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + target = TransportTarget("telegram", "123") + + class HostileReceipt(TransportReceipt): + def __getattribute__(self, name): + try: + armed = object.__getattribute__(self, "_armed") + except AttributeError: + armed = False + if armed and name not in {"__class__", "_armed"}: + raise AssertionError("hostile receipt attribute was read") + return super().__getattribute__(name) + + subclass_execution = executions.create_execution("receipt-subclass", source="direct") + subclass_attempt = executions.preregister_receipt_plan( + subclass_execution["id"], fire_identity="receipt-subclass-fire", + components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + )[0] + hostile_receipt = HostileReceipt(outcome="unknown", requested_target=target) + object.__setattr__(hostile_receipt, "_armed", True) + with __import__("pytest").raises(ValueError, match="TransportReceipt"): + executions.observe_transport_unknown(subclass_attempt["id"], hostile_receipt) + + class HostileText(str): + def __hash__(self): + raise AssertionError("hostile outcome was hashed") + + def __eq__(self, _other): + raise AssertionError("hostile outcome was compared") + + record_execution = executions.create_execution("mutated-outcome", source="direct") + record_attempt = executions.preregister_receipt_plan( + record_execution["id"], fire_identity="mutated-outcome-fire", + components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + )[0] + delivered = TransportReceipt( + outcome="delivered", requested_target=target, actual_target=target, + provider_message_id="provider-id", + ) + object.__setattr__(delivered, "outcome", HostileText("delivered")) + with __import__("pytest").raises(ValueError, match="outcome"): + executions.record_transport_receipt(record_attempt["id"], delivered) + + class HostileDateTime(datetime): + def astimezone(self, *_args, **_kwargs): + raise AssertionError("hostile datetime conversion was called") + + def isoformat(self, *_args, **_kwargs): + raise AssertionError("hostile datetime formatting was called") + + observe_execution = executions.create_execution("mutated-observed", source="direct") + observe_attempt = executions.preregister_receipt_plan( + observe_execution["id"], fire_identity="mutated-observed-fire", + components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + )[0] + unknown = TransportReceipt(outcome="unknown", requested_target=target) + object.__setattr__(unknown, "observed_at", HostileDateTime.now(timezone.utc)) + with __import__("pytest").raises(ValueError, match="timezone-aware"): + executions.observe_transport_unknown(observe_attempt["id"], unknown) + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + rows = conn.execute( + "SELECT outcome, observed_at FROM delivery_attempts ORDER BY id" + ).fetchall() + assert rows and all(row == ("unknown", None) for row in rows) + + +def test_mutated_typed_target_fields_cannot_mark_attempt_delivered(monkeypatch, tmp_path): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + for target_field in ("requested_target", "actual_target"): + execution = executions.create_execution( + f"mutated-{target_field}", source="builtin", + ) + attempts = executions.preregister_receipt_plan( + execution["id"], fire_identity=f"fire-{target_field}", components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + ) + receipt = TransportReceipt( + outcome="delivered", + provider_message_id=f"provider-{target_field}", + requested_target=TransportTarget("telegram", "123"), + actual_target=TransportTarget("telegram", "123"), + ) + object.__setattr__(getattr(receipt, target_field), "thread_id", "") + + with __import__("pytest").raises(ValueError, match="thread_id"): + executions.record_transport_receipt(attempts[0]["id"], receipt) + assert executions.receipt_summary(execution["id"])["unknown"] == 1 + + +def test_mutated_failed_receipt_cannot_persist_provider_evidence_or_unknown_kind( + monkeypatch, tmp_path, +): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + target = TransportTarget("telegram", "123") + mutations = ( + ("provider_message_id", "provider-id", "provider"), + ("actual_target", target, "actual_target"), + ("failure_kind", "unbounded_kind", "failure_kind"), + ) + + for ordinal, (field, value, match) in enumerate(mutations): + execution = executions.create_execution( + f"failed-boundary-{ordinal}", source="builtin", + ) + attempts = executions.preregister_receipt_plan( + execution["id"], fire_identity=f"failed-fire-{ordinal}", components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + ) + receipt = TransportReceipt( + outcome="failed", + requested_target=target, + component="text", + ordinal=0, + failure_kind="pre_dispatch", + ) + object.__setattr__(receipt, field, value) + + with __import__("pytest").raises(ValueError, match=match): + executions.record_transport_receipt(attempts[0]["id"], receipt) + assert executions.receipt_summary(execution["id"])["unknown"] == 1 + + +def test_mutated_delivered_receipt_cannot_persist_failure_kind(monkeypatch, tmp_path): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("delivered-boundary", source="builtin") + attempts = executions.preregister_receipt_plan( + execution["id"], fire_identity="delivered-fire", components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + ) + target = TransportTarget("telegram", "123") + receipt = TransportReceipt( + outcome="delivered", + provider_message_id="provider-id", + requested_target=target, + actual_target=target, + component="text", + ordinal=0, + ) + object.__setattr__(receipt, "failure_kind", "pre_dispatch") + + with __import__("pytest").raises(ValueError, match="failure_kind"): + executions.record_transport_receipt(attempts[0]["id"], receipt) + assert executions.receipt_summary(execution["id"])["unknown"] == 1 + + +def test_receipt_plan_and_persistence_reject_boolean_ordinals(monkeypatch, tmp_path): + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + rejected = executions.create_execution("bool-plan", source="builtin") + with __import__("pytest").raises(ValueError, match="ordinal"): + executions.preregister_receipt_plan( + rejected["id"], fire_identity="bool-plan-fire", components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": True, "content": "bounded", + }], + ) + + execution = executions.create_execution("bool-receipt", source="builtin") + attempts = executions.preregister_receipt_plan( + execution["id"], fire_identity="bool-receipt-fire", components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }], + ) + target = TransportTarget("telegram", "123") + receipt = TransportReceipt( + outcome="delivered", provider_message_id="provider-1", + requested_target=target, actual_target=target, ordinal=0, + ) + object.__setattr__(receipt, "ordinal", False) + with __import__("pytest").raises(ValueError, match="ordinal"): + executions.record_transport_receipt(attempts[0]["id"], receipt) + assert executions.receipt_summary(execution["id"])["unknown"] == 1 + + +def test_partial_confirmation_blocks_a_second_plan_for_the_same_fire(monkeypatch, tmp_path): + """Unknown never authorizes replay after any component may have dispatched.""" + from gateway.platforms.base import TransportReceipt, TransportTarget + + executions = _point_ledger(monkeypatch, tmp_path) + first_execution = executions.create_execution("job-partial", source="builtin") + plan = [ + { + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "report", + }, + { + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "media", "ordinal": 0, "content": "/private/report.pdf", + }, + ] + attempts = executions.preregister_receipt_plan( + first_execution["id"], fire_identity="fire-partial", components=plan, + ) + target = TransportTarget("telegram", "123") + assert executions.record_transport_receipt( + attempts[0]["id"], + TransportReceipt( + outcome="delivered", provider_message_id="provider-1", + requested_target=target, actual_target=target, + component="text", ordinal=0, + ), + ) is True + + recovery = executions.create_execution("job-partial", source="recovery") + with __import__("pytest").raises(ValueError, match="already attempted"): + executions.preregister_receipt_plan( + recovery["id"], fire_identity="fire-partial", components=plan, + ) + + +def test_receipt_schema_upgrade_is_singleton_and_idempotent(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + legacy = executions.create_execution("legacy-receipt", source="builtin") + executions.finish_execution(legacy["id"], success=True) + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + conn.execute("UPDATE executions SET receipt_state=NULL WHERE id=?", (legacy["id"],)) + conn.execute("DROP TABLE receipt_schema") + conn.execute("CREATE TABLE receipt_schema(version INTEGER PRIMARY KEY)") + conn.execute("INSERT INTO receipt_schema(version) VALUES (1), (2)") + + assert executions.receipt_summary(legacy["id"])["unknown"] == 1 + assert executions.receipt_summary(legacy["id"])["unknown"] == 1 + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + columns = [row[1] for row in conn.execute("PRAGMA table_info(receipt_schema)")] + rows = list(conn.execute("SELECT singleton, version FROM receipt_schema")) + synthetic = conn.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] + assert columns == ["singleton", "version"] + assert rows == [(1, executions._RECEIPT_SCHEMA_VERSION)] + assert synthetic == 0 + + +def test_execution_and_receipt_database_never_persists_payloads_paths_or_raw_errors(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("privacy-job", source="builtin") + executions.preregister_receipt_plan( + execution["id"], fire_identity="privacy-fire", + components=[ + { + "target": {"platform": "matrix", "chat_id": "!room:example.org"}, + "component": "text", "ordinal": 0, + "content": "PRIVATE_REPORT_BODY_SENTINEL", + }, + { + "target": {"platform": "matrix", "chat_id": "!room:example.org"}, + "component": "media", "ordinal": 0, + "content": "/private/media/PAYSLIP_SENTINEL.pdf", + }, + ], + ) + failed = executions.finish_execution( + execution["id"], success=False, + error="RAW_PROVIDER_EXCEPTION_SENTINEL user@example.org", + ) + assert failed["error"] is None + assert failed["error_kind"] == "execution_failed" + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + dump = "\n".join(conn.iterdump()) + assert "PRIVATE_REPORT_BODY_SENTINEL" not in dump + assert "PAYSLIP_SENTINEL" not in dump + assert "RAW_PROVIDER_EXCEPTION_SENTINEL" not in dump + assert "user@example.org" not in dump + def test_execution_can_be_loaded_by_exact_attempt_id(monkeypatch, tmp_path): executions = _point_ledger(monkeypatch, tmp_path) first = executions.create_execution("same-job", source="builtin") @@ -179,6 +905,46 @@ def test_retention_bounds_terminal_history_but_preserves_inflight(monkeypatch, t assert executions.latest_execution("live")["status"] == "running" +def test_receipt_preregistration_is_concurrently_idempotent(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + execution = executions.create_execution("concurrent-plan", source="builtin") + plan = [{ + "target": {"platform": "matrix", "chat_id": "!room:example.org"}, + "component": "text", "ordinal": 0, "content": "same content", + }] + + def register(): + return executions.preregister_receipt_plan( + execution["id"], fire_identity="same-fire", components=plan, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _index: register(), range(2))) + assert results[0] == results[1] + + +def test_execution_retention_cascades_receipt_rows(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + monkeypatch.setattr(executions, "MAX_TERMINAL_EXECUTIONS", 1) + old = executions.create_execution("old-receipt", source="builtin") + executions.preregister_receipt_plan( + old["id"], fire_identity="old-fire", + components=[{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "old", + }], + ) + executions.finish_execution(old["id"], success=True) + new = executions.create_execution("new-receipt", source="builtin") + executions.finish_execution(new["id"], success=True) + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + assert conn.execute( + "SELECT COUNT(*) FROM delivery_targets WHERE execution_id=?", (old["id"],) + ).fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM delivery_components").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 0 + def test_recently_finished_long_running_execution_survives_retention( monkeypatch, tmp_path ): @@ -218,7 +984,9 @@ def test_cron_runs_cli_prints_execution_history(monkeypatch, tmp_path, capsys): output = capsys.readouterr().out assert row["id"] in output assert "failed" in output - assert "boom" in output + assert "Failure kind: execution_failed" in output + assert "boom" not in output + assert "Receipt: delivered=0 failed=0 unknown=0" in output def test_quick_backup_includes_execution_ledger(): @@ -227,14 +995,15 @@ def test_quick_backup_includes_execution_ledger(): assert "cron/executions.db" in _QUICK_STATE_FILES -def test_failed_execution_keeps_error(monkeypatch, tmp_path): +def test_failed_execution_keeps_only_bounded_error_category(monkeypatch, tmp_path): executions = _point_ledger(monkeypatch, tmp_path) record = executions.create_execution("job-2", source="external") failed = executions.finish_execution(record["id"], success=False, error="provider exploded") assert failed["status"] == "failed" - assert failed["error"] == "provider exploded" + assert failed["error"] is None + assert failed["error_kind"] == "execution_failed" def test_recovery_does_not_mark_live_process_execution_unknown(monkeypatch, tmp_path): @@ -291,7 +1060,8 @@ def test_restart_marks_interrupted_execution_unknown_without_requeue(tmp_path): assert records[0]["id"] == execution_id assert records[0]["status"] == "unknown" assert records[0]["finished_at"] - assert "restart" in records[0]["error"].lower() + assert records[0]["error"] is None + assert records[0]["error_kind"] == "interrupted" # Recovery only classifies the old attempt. It must not manufacture a new # claimed record (which would imply an automatic retry). assert [r["status"] for r in records] == ["unknown"] @@ -362,6 +1132,159 @@ def fake_run_job(job, *, defer_agent_teardown=None, execution_id=None, **_kw): assert events[-1][2]["success"] is True +def test_direct_run_passes_one_execution_and_fire_identity_to_delivery(monkeypatch): + """Manual/direct fires must not lose the durable receipt identity.""" + import cron.scheduler as scheduler + from cron.executions import scheduled_fire_identity + + captured = {} + fire_at = "2026-08-22T19:00:00+00:00" + + def create(job_id, *, source, **kwargs): + captured["created"] = (job_id, source, kwargs.get("fire_identity")) + return {"id": "exec-direct", "fire_identity": kwargs.get("fire_identity")} + + monkeypatch.setattr(scheduler, "create_execution", create) + def bind(execution_id, fire_identity): + captured["bound"] = (execution_id, fire_identity) + return {"id": execution_id, "fire_identity": fire_identity} + + monkeypatch.setattr(scheduler, "bind_execution_fire_identity", bind) + monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_a, **_kw: {}) + monkeypatch.setattr(scheduler, "finish_execution", lambda *_a, **_kw: None) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr(scheduler, "heartbeat_fire_claim", lambda *_a, **_kw: True) + monkeypatch.setattr( + scheduler, + "fire_claim_fence", + lambda *_a, **_kw: __import__("contextlib").nullcontext(True), + ) + monkeypatch.setattr( + scheduler, "run_job", + lambda _job, *, defer_agent_teardown=None, **_kw: (True, "output", "response", None), + ) + monkeypatch.setattr(scheduler, "save_job_output", lambda *_a: None) + monkeypatch.setattr(scheduler, "mark_job_run", lambda *_a, **_kw: None) + + def deliver(_job, _content, **kwargs): + captured.update(kwargs) + return None + + monkeypatch.setattr(scheduler, "_deliver_result", deliver) + + assert scheduler.run_one_job({ + "id": "manual-identity", + "deliver": "local", + "fire_claim": {"by": "manual-owner", "at": fire_at, "fire_at": fire_at}, + }) is True + expected = scheduled_fire_identity("manual-identity", fire_at) + assert captured == { + "created": ("manual-identity", "direct", None), + "bound": ("exec-direct", expected), + "adapters": None, + "loop": None, + "execution_id": "exec-direct", + "fire_identity": expected, + "for_failure": False, + } + + +def test_existing_execution_binding_is_verified_from_ledger_before_running( + monkeypatch, tmp_path +): + """A job snapshot cannot prove that its persisted execution is bound.""" + import contextlib + import cron.scheduler as scheduler + + executions = _point_ledger(monkeypatch, tmp_path) + fire_at = "2026-08-22T19:00:00+00:00" + expected = executions.scheduled_fire_identity("existing-direct", fire_at) + execution = executions.create_execution("existing-direct", source="direct") + assert execution["fire_identity"] == execution["id"] + + def mark_running(execution_id): + persisted = executions.list_executions(job_id="existing-direct") + assert persisted[0]["id"] == execution_id + assert persisted[0]["fire_identity"] == expected + + monkeypatch.setattr(scheduler, "mark_execution_running", mark_running) + monkeypatch.setattr(scheduler, "finish_execution", lambda *_a, **_kw: None) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr(scheduler, "heartbeat_fire_claim", lambda *_a, **_kw: True) + monkeypatch.setattr( + scheduler, + "fire_claim_fence", + lambda *_a, **_kw: contextlib.nullcontext(True), + ) + monkeypatch.setattr( + scheduler, "run_job", + lambda _job, *, defer_agent_teardown=None, **_kw: ( + True, "output", "response", None, + ), + ) + monkeypatch.setattr(scheduler, "save_job_output", lambda *_a: None) + monkeypatch.setattr(scheduler, "_deliver_result", lambda *_a, **_kw: None) + monkeypatch.setattr(scheduler, "mark_job_run", lambda *_a, **_kw: None) + + assert scheduler.run_one_job({ + "id": "existing-direct", + "execution_id": execution["id"], + # Adversarial stale/forged snapshot: it claims the expected identity, + # while the authoritative ledger is still default-bound to execution_id. + "fire_identity": expected, + "fire_claim": { + "by": "manual-owner", "at": fire_at, "fire_at": fire_at, + }, + }) is True + + +def test_builtin_tick_binds_due_timestamp_through_claim_to_execution(monkeypatch): + import cron.scheduler as scheduler + from cron.executions import scheduled_fire_identity + + due_at = "2026-08-22T19:00:00+00:00" + acquired_at = "2026-08-22T20:00:00+00:00" + due_job = { + "id": "builtin-fire-identity", "name": "identity", + "schedule": {"kind": "interval", "seconds": 3600}, + "next_run_at": due_at, "enabled": True, + } + captured = {} + + def create(job_id, *, source, **kwargs): + captured["created"] = (job_id, source, kwargs.get("fire_identity")) + return {"id": "exec-builtin", "fire_identity": kwargs.get("fire_identity")} + + def bind(execution_id, fire_identity): + captured["bound"] = (execution_id, fire_identity) + return {"id": execution_id, "fire_identity": fire_identity} + + def run(claimed_job, **_kwargs): + captured["run"] = claimed_job + return True + + monkeypatch.setattr(scheduler, "get_due_jobs", lambda: [due_job]) + monkeypatch.setattr(scheduler, "advance_next_runs", lambda _ids: None) + monkeypatch.setattr(scheduler, "load_config", lambda: {}) + monkeypatch.setattr(scheduler, "create_execution", create) + monkeypatch.setattr(scheduler, "bind_execution_fire_identity", bind, raising=False) + monkeypatch.setattr( + scheduler, "claim_job_for_fire", + lambda _jid, **_kwargs: dict( + due_job, + next_run_at="2026-08-22T21:00:00+00:00", + fire_claim={"by": "builtin-owner", "at": acquired_at, "fire_at": acquired_at}, + ), + ) + monkeypatch.setattr(scheduler, "run_one_job", run) + + assert scheduler.tick(verbose=False, sync=True) == 1 + expected = scheduled_fire_identity(due_job["id"], acquired_at) + assert captured["created"] == (due_job["id"], "builtin", None) + assert captured["bound"] == ("exec-builtin", expected) + assert captured["run"]["fire_identity"] == expected + + def test_provider_start_recovers_interrupted_records_before_tick(monkeypatch): import cron.scheduler_provider as provider diff --git a/tests/cron/test_fire_forward_failure_stamp.py b/tests/cron/test_fire_forward_failure_stamp.py index cb205f8fb69e..e159a62fa318 100644 --- a/tests/cron/test_fire_forward_failure_stamp.py +++ b/tests/cron/test_fire_forward_failure_stamp.py @@ -86,4 +86,7 @@ def test_cronjob_list_carries_last_fire_error(self, tmp_cron_dir): job = create_job(prompt="Daily invoice triage", schedule="every 1h") note_fire_forward_failure(job["id"], "gateway unreachable") formatted = _format_job(get_job(job["id"])) - assert formatted["last_fire_error"]["detail"] == "gateway unreachable" + assert formatted["last_fire_error"] == { + "at": get_job(job["id"])["last_fire_error"]["at"], + "error_kind": "fire_forward_failed", + } diff --git a/tests/cron/test_inflight_stale_guard.py b/tests/cron/test_inflight_stale_guard.py index c84b29949413..4ca8d587ce00 100644 --- a/tests/cron/test_inflight_stale_guard.py +++ b/tests/cron/test_inflight_stale_guard.py @@ -342,6 +342,11 @@ def test_tick_sweeps_then_dispatches_the_previously_wedged_job(self, tmp_path): patch.object(sched, "advance_next_runs"), \ patch.object(sched, "mark_job_run"), \ patch.object(sched, "create_execution", return_value={"id": "exec-1"}), \ + patch.object( + sched, + "bind_execution_fire_identity", + side_effect=lambda eid, fire: {"id": eid, "fire_identity": fire}, + ), \ patch.object(sched, "finish_execution"), \ patch.object(sched, "run_one_job", return_value=True): n = sched.tick(verbose=False) diff --git a/tests/cron/test_jobs_file_ownership.py b/tests/cron/test_jobs_file_ownership.py index e1dce9bee42f..15382786003d 100644 --- a/tests/cron/test_jobs_file_ownership.py +++ b/tests/cron/test_jobs_file_ownership.py @@ -215,26 +215,25 @@ def test_successful_tick_clears_error(self, cron_store, monkeypatch): class TestCronStatusSurfacesError: - def test_status_shows_last_error_and_permission_hint(self, monkeypatch, capsys): + def test_status_redacts_last_error_and_keeps_permission_hint(self, monkeypatch, capsys): from hermes_cli import cron as cron_cli + sentinel = ( + "RuntimeError: Failed to read cron database: [Errno 13] " + "Permission denied: '/private/jobs.json' user@example.org" + ) monkeypatch.setattr("hermes_cli.gateway.find_gateway_pids", lambda: [4321]) monkeypatch.setattr(jobs, "get_ticker_heartbeat_age", lambda: 5.0) # alive monkeypatch.setattr(jobs, "get_ticker_success_age", lambda: 9_999.0) # failing - monkeypatch.setattr( - jobs, - "get_ticker_last_error", - lambda: ( - "RuntimeError: Failed to read cron database: " - "[Errno 13] Permission denied: '/opt/data/cron/jobs.json'" - ), - ) + monkeypatch.setattr(jobs, "get_ticker_last_error", lambda: sentinel) monkeypatch.setattr("cron.jobs.list_jobs", lambda **k: []) cron_cli.cron_status() out = capsys.readouterr().out - assert "Last tick error:" in out - assert "Permission denied" in out + assert "Last tick error: permission_denied" in out + assert sentinel not in out + assert "/private/jobs.json" not in out + assert "user@example.org" not in out # The permission-specific hint must point at the ownership fix. assert "docker exec -u" in out diff --git a/tests/cron/test_parallel_pool.py b/tests/cron/test_parallel_pool.py index 9853dbc2291e..8f9c8323e9eb 100644 --- a/tests/cron/test_parallel_pool.py +++ b/tests/cron/test_parallel_pool.py @@ -119,7 +119,16 @@ def submit(self, callback): sched, "claim_job_for_fire", lambda job_id, **kwargs: claim_calls.append((job_id, kwargs)) - or {**job, "fire_claim": {"by": "worker-owner", "at": "now"}}, + or {**job, "fire_claim": { + "by": "worker-owner", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }}, + ) + monkeypatch.setattr( + sched, + "bind_execution_fire_identity", + lambda eid, fire: {"id": eid, "fire_identity": fire}, ) monkeypatch.setattr(sched, "run_one_job", lambda *_a, **_kw: True) @@ -164,7 +173,7 @@ def test_create_execution_failure_does_not_wedge_running_set(self, tmp_path, mon called = [] - def create_execution_side_effect(job_id, source): + def create_execution_side_effect(job_id, source, **_kwargs): if job_id == "failing-job": raise RuntimeError("execution ledger unavailable") return {"id": f"{job_id}-execution"} @@ -182,11 +191,21 @@ def create_execution_side_effect(job_id, source): sched, "claim_job_for_fire", lambda job_id, **_kw: dict( - healthy_job, fire_claim={"by": "test-owner", "at": "now"} + healthy_job, + fire_claim={ + "by": "test-owner", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }, ) if job_id == "healthy-job" else None, ) + monkeypatch.setattr( + sched, + "bind_execution_fire_identity", + lambda eid, fire: {"id": eid, "fire_identity": fire}, + ) monkeypatch.setattr(sched, "mark_execution_running", lambda *_a, **_kw: {}) monkeypatch.setattr(sched, "heartbeat_fire_claim", lambda *_a, **_kw: True) diff --git a/tests/cron/test_relay_fronted_delivery.py b/tests/cron/test_relay_fronted_delivery.py index 76a7520f9c43..0cf324ec1e2a 100644 --- a/tests/cron/test_relay_fronted_delivery.py +++ b/tests/cron/test_relay_fronted_delivery.py @@ -28,6 +28,7 @@ _resolve_delivery_targets, ) from gateway.config import HomeChannel, Platform +from gateway.platforms.base import SendResult, TransportReceipt, TransportTarget def _gateway_config_with_home(platform=Platform.DISCORD, chat_id="1517373704248758474", @@ -142,7 +143,15 @@ def fake_run_coro(coro, _loop): router = MagicMock() async def _deliver_to_platform(target, content, metadata): - return {"success": True, "raw_response": None} + receipt_target = TransportTarget("discord", "123") + return SendResult( + success=True, + message_id="relay-message", + receipts=(TransportReceipt( + outcome="delivered", provider_message_id="relay-message", + requested_target=receipt_target, actual_target=receipt_target, + ),), + ) router._deliver_to_platform = _deliver_to_platform diff --git a/tests/cron/test_restart_safe_worker.py b/tests/cron/test_restart_safe_worker.py index 1be7e30654ab..ad3653212722 100644 --- a/tests/cron/test_restart_safe_worker.py +++ b/tests/cron/test_restart_safe_worker.py @@ -74,7 +74,8 @@ def test_genuine_external_worker_crash_is_recovered_unknown( assert execution_ledger.recover_interrupted_executions() == 1 recovered = execution_ledger.latest_execution("job-crash") assert recovered["status"] == "unknown" - assert "whether side effects ran is unknown" in recovered["error"] + assert recovered["error"] is None + assert recovered["error_kind"] == "interrupted" @pytest.mark.linux_only diff --git a/tests/cron/test_run_one_job.py b/tests/cron/test_run_one_job.py index 9b241de1b63e..4cd85ccb5dad 100644 --- a/tests/cron/test_run_one_job.py +++ b/tests/cron/test_run_one_job.py @@ -78,6 +78,81 @@ def test_run_one_job_success_sequence(monkeypatch): assert calls[-1] == ("mark", "j2", True) +def test_run_one_job_uses_unknown_delivery_outcome_from_receipt_ledger(monkeypatch): + finished = [] + monkeypatch.setattr( + s, "create_execution", lambda *_a, **_kw: {"id": "exec-unknown"} + ) + monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) + monkeypatch.setattr( + s, "run_job", + lambda *_a, **_kw: (True, "out", "final response", None), + ) + monkeypatch.setattr(s, "save_job_output", lambda *_a: None) + monkeypatch.setattr( + s, + "_deliver_result", + lambda *_a, **_kw: "bot-chat delivery confirmation unavailable", + ) + monkeypatch.setattr(s, "mark_job_run", lambda *_a, **_kw: None) + monkeypatch.setattr( + s, + "receipt_summary", + lambda _execution_id: { + "delivered": 0, "failed": 0, "unknown": 1, "targets_delivered": 0, + }, + raising=False, + ) + monkeypatch.setattr( + s, + "finish_execution", + lambda *args, **kwargs: finished.append((args, kwargs)), + ) + + assert s.run_one_job({ + "id": "j-unknown", "name": "bot", "deliver": "bot-chat:research", + }) is True + assert finished[-1][1]["delivery_outcome"] == "unknown" + + +def test_run_one_job_keeps_concrete_delivery_error_failed_despite_text_ack( + monkeypatch, +): + finished = [] + monkeypatch.setattr( + s, "create_execution", lambda *_a, **_kw: {"id": "exec-partial"} + ) + monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) + monkeypatch.setattr( + s, "run_job", lambda *_a, **_kw: (True, "out", "final response", None) + ) + monkeypatch.setattr(s, "save_job_output", lambda *_a: None) + monkeypatch.setattr( + s, "_deliver_result", lambda *_a, **_kw: "media path policy rejected attachment" + ) + monkeypatch.setattr(s, "mark_job_run", lambda *_a, **_kw: None) + monkeypatch.setattr( + s, + "receipt_summary", + lambda _execution_id: { + "delivered": 1, "failed": 0, "unknown": 0, "targets_delivered": 1, + }, + raising=False, + ) + monkeypatch.setattr( + s, + "finish_execution", + lambda *args, **kwargs: finished.append((args, kwargs)), + ) + + assert s.run_one_job({ + "id": "j-partial", "name": "partial", "deliver": "telegram", + }) is True + assert finished[-1][1]["delivery_outcome"] == "failed" + + def test_run_one_job_exception_delivers_failure_alert(monkeypatch): """An exception escaping the run body must not become a silent error row.""" delivered = [] diff --git a/tests/cron/test_scheduler.py b/tests/cron/test_scheduler.py index e7f99c2594f9..94e43b92b4b1 100644 --- a/tests/cron/test_scheduler.py +++ b/tests/cron/test_scheduler.py @@ -1,10 +1,12 @@ """Tests for cron/scheduler.py — origin resolution, delivery routing, and error logging.""" +import asyncio import contextlib import itertools import json import logging import os +from types import SimpleNamespace from unittest.mock import AsyncMock, patch, MagicMock import pytest @@ -390,8 +392,10 @@ def test_relay_fronted_home_uses_relay_config_and_live_adapter(self, monkeypatch relay = MagicMock() relay.fronts_platform.side_effect = lambda platform: platform == Platform.SLACK - relay.send_for_platform = AsyncMock(return_value=MagicMock(success=True)) - relay.send_voice = AsyncMock(return_value=MagicMock(success=True)) + relay.send_for_platform = AsyncMock( + return_value=SimpleNamespace(success=True) + ) + relay.send_voice = AsyncMock(return_value=SimpleNamespace(success=True)) relay.supports_inchannel_continuable = False # Not a real RelayAdapter: keep the auto-created accessor from # shadowing the scalar False (MagicMock fabricates truthy callables). @@ -445,7 +449,8 @@ def fake_run_coro(coro, _loop): loop=loop, ) - assert result is None + assert result is not None + assert "media acknowledgement" in result relay.send_for_platform.assert_awaited_once() args = relay.send_for_platform.await_args.args assert args[:3] == (Platform.SLACK, "D123", "scheduled result") @@ -466,8 +471,8 @@ def test_live_adapter_sends_media_as_attachments(self, tmp_path, monkeypatch): media_path = self._safe_media_path(tmp_path, monkeypatch, "cron-voice.mp3") adapter = AsyncMock() - adapter.send.return_value = MagicMock(success=True) - adapter.send_voice.return_value = MagicMock(success=True) + adapter.send.return_value = SimpleNamespace(success=True) + adapter.send_voice.return_value = SimpleNamespace(success=True) pconfig = MagicMock() pconfig.enabled = True @@ -1801,7 +1806,7 @@ def _run_with_loop(adapter, chat_id, media_files, metadata, job): def fake_run_coro(coro, _loop): coro.close() completed = Future() - completed.set_result(MagicMock(success=True)) + completed.set_result(SimpleNamespace(success=True)) return completed with patch("asyncio.run_coroutine_threadsafe", side_effect=fake_run_coro): @@ -1942,20 +1947,10 @@ def mock_run_job(job, *, defer_agent_teardown=None, **_kw): class TestDeliverResultTimeoutCancelsFuture: - """When future.result(timeout=60) raises TimeoutError in the live adapter - delivery path, the outcome depends on whether the coroutine was already - running. future.cancel() returning False means it is in flight on the wire - (cannot be un-sent) → treat as DELIVERED and skip the standalone fallback to - avoid a duplicate (#38922). future.cancel() returning True means it never - started (wedged loop) → nothing was sent, so fall through to standalone or - the message is silently dropped. Regression for #38922. - """ + """A live confirmation timeout is unknown, never receipt evidence.""" - def test_live_adapter_timeout_assumes_delivered_no_duplicate(self): - """End-to-end: live adapter confirmation times out past the 60s budget. - The fix (#38922) treats the send as already-dispatched/delivered and - does NOT run the standalone fallback — otherwise the message is sent - twice.""" + def test_live_adapter_timeout_is_unknown_without_fallback(self): + """Neither Future.cancel() result can prove dispatch or delivery.""" from gateway.config import Platform from concurrent.futures import Future @@ -1973,9 +1968,8 @@ def test_live_adapter_timeout_assumes_delivered_no_duplicate(self): # A real concurrent.futures.Future, but we override .result() to raise # TimeoutError exactly like the 60s wait firing in production. We make - # .cancel() return False to simulate the coroutine being ALREADY RUNNING - # on the gateway loop (in flight on the wire) — the case where the send - # cannot be un-sent and a standalone resend would be a duplicate. + # cancel() returning False only says cancellation lost a race; it does + # not acknowledge a provider write or identify a recipient. captured_future = Future() cancel_calls = [] @@ -2009,12 +2003,14 @@ def fake_run_coro(coro, _loop): loop=loop, ) - # 1. cancel() was attempted (returned False = in flight). + # 1. Best-effort cancellation may still be attempted, but is not used + # as receipt evidence. assert cancel_calls == [True], "future.cancel() should be attempted on TimeoutError" - # 2. Delivery is reported successful (no error string returned). - assert result is None, f"expected successful delivery, got error: {result!r}" - # 3. The standalone fallback must NOT run — that is the #38922 fix: - # an in-flight confirmation timeout is assume-delivered, not a resend. + # 2. The caller sees an ambiguous/unknown error, not success. + assert result is not None + assert "confirmation timed out" in result + # 3. No same-identity fallback after unknown: it could duplicate a + # request that reached Telegram. standalone_send.assert_not_awaited() @@ -2073,12 +2069,22 @@ def fake_run_coro(coro, _loop): ) return result, standalone_send - def test_none_result_falls_through_to_standalone(self): - """send() returning None must trigger the standalone fallback, not a - silent "delivered" log.""" + def test_none_result_is_unknown_without_standalone_retry(self): + """No response cannot prove the adapter did not dispatch the write.""" result, standalone_send = self._run(None) - assert result is None, f"standalone should have delivered, got: {result!r}" - standalone_send.assert_awaited_once() + assert result is not None + assert "unconfirmed" in result + standalone_send.assert_not_awaited() + + def test_legacy_success_without_typed_receipt_is_unknown_without_fallback(self): + """A legacy success bit has no provider-issued delivery evidence.""" + from gateway.platforms.base import SendResult + + result, standalone_send = self._run(SendResult(success=True, message_id="legacy")) + + assert result is not None + assert "unknown" in result + standalone_send.assert_not_awaited() class TestDeliverOriginUnresolvableIsLocal: @@ -2416,7 +2422,7 @@ def fake_run_coro(coro, _loop): def _slack_adapter(self, supports_inchannel=True, with_store=True): adapter = AsyncMock() - adapter.send.return_value = MagicMock( + adapter.send.return_value = SimpleNamespace( success=True, message_id="msg_1", raw_response=None, ) # Capability flag read via getattr in the scheduler. @@ -2522,8 +2528,18 @@ def __init__(self, *a, **k): pass async def _deliver_to_platform(self, target, text, metadata): + from gateway.platforms.base import SendResult, TransportReceipt, TransportTarget + captured["target"] = target - return {"success": True, "message_id": "msg_1"} + receipt_target = TransportTarget("slack", "C123") + return SendResult( + success=True, + message_id="msg_1", + receipts=(TransportReceipt( + outcome="delivered", provider_message_id="msg_1", + requested_target=receipt_target, actual_target=receipt_target, + ),), + ) adapter = self._slack_adapter(supports_inchannel=True) origin_with_thread = { @@ -2733,6 +2749,10 @@ def test_first_target_failure_does_not_crash_loop(self): assert mock_pool.submit.call_count == 2, ( f"expected 2 delivery attempts, got {mock_pool.submit.call_count}" ) + assert all( + not any(asyncio.iscoroutine(arg) for arg in call.args) + for call in mock_pool.submit.call_args_list + ), "create the coroutine inside the worker, never before executor ownership" # First target's failure is surfaced in the returned error string. assert result is not None assert "a@example.com" in result diff --git a/tests/cron/test_scheduler_provider.py b/tests/cron/test_scheduler_provider.py index 6bf710e4ed30..764da6663e11 100644 --- a/tests/cron/test_scheduler_provider.py +++ b/tests/cron/test_scheduler_provider.py @@ -18,7 +18,7 @@ """ import threading import time -from unittest.mock import patch +from unittest.mock import Mock, patch def _wait_until(predicate, timeout=10.0, interval=0.005): @@ -374,7 +374,11 @@ def test_fire_due_default_claims_then_runs(monkeypatch): jobs, "claim_job_for_fire", lambda jid, **kw: claims.append((jid, kw)) - or {"id": jid, "name": "t", "fire_claim": {"by": "exact-owner"}}, + or {"id": jid, "name": "t", "fire_claim": { + "by": "exact-owner", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }}, raising=False, ) monkeypatch.setattr( @@ -399,13 +403,22 @@ def test_claim_fire_persists_attempt_before_fire_claimed(monkeypatch): jobs, "claim_job_for_fire", lambda jid, **kwargs: events.append("claim") - or {"id": jid, "fire_claim": {"by": "owner"}}, + or {"id": jid, "fire_claim": { + "by": "owner", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }}, ) monkeypatch.setattr( executions, "create_execution", lambda jid, source: events.append("ledger") or {"id": "exec-1"}, ) + monkeypatch.setattr( + executions, + "bind_execution_fire_identity", + lambda eid, fire: events.append("bind") or {"id": eid, "fire_identity": fire}, + ) monkeypatch.setattr( sched, "run_one_job", @@ -415,29 +428,239 @@ def test_claim_fire_persists_attempt_before_fire_claimed(monkeypatch): provider = InProcessCronScheduler() claimed = provider.claim_fire("j1") - assert events == ["ledger", "claim"] + assert events == ["ledger", "claim", "bind"] assert claimed is not None assert claimed["execution_id"] == "exec-1" assert provider.fire_claimed(claimed) is True - assert events == ["ledger", "claim", ("run", "exec-1")] + assert events == ["ledger", "claim", "bind", ("run", "exec-1")] + + +def test_claim_fire_binds_scheduled_fire_before_provider_dispatch(monkeypatch): + import cron.executions as executions + import cron.jobs as jobs + from cron.scheduler_provider import InProcessCronScheduler + + created = [] + monkeypatch.setattr(jobs, "get_job", lambda _jid: { + "id": "j1", "next_run_at": "2026-08-22T19:00:00+00:00", + }) + monkeypatch.setattr(jobs, "claim_job_for_fire", lambda jid, **_kwargs: { + "id": jid, "fire_claim": { + "by": "owner", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }, + }) + + def create(jid, *, source, **kwargs): + created.append((jid, source, kwargs.get("fire_identity"))) + return {"id": "exec-1", "fire_identity": kwargs.get("fire_identity")} + + monkeypatch.setattr(executions, "create_execution", create) + monkeypatch.setattr( + executions, "bind_execution_fire_identity", + lambda eid, fire: {"id": eid, "fire_identity": fire}, + ) + claimed = InProcessCronScheduler().claim_fire("j1") + + assert created[0][2] is None + assert claimed["fire_identity"] == executions.scheduled_fire_identity( + "j1", "2026-08-22T19:00:00+00:00", + ) + + +def test_claim_fire_binds_identity_from_acquired_claim_not_stale_pre_read(monkeypatch): + import cron.executions as executions + import cron.jobs as jobs + from cron.scheduler_provider import InProcessCronScheduler + + events = [] + stale_due = "2026-08-22T19:00:00+00:00" + acquired_at = "2026-08-22T20:00:00+00:00" + monkeypatch.setattr(jobs, "get_job", lambda _jid: { + "id": "j1", "next_run_at": stale_due, + }) + monkeypatch.setattr(jobs, "claim_job_for_fire", lambda jid, **_kwargs: ( + events.append("claim") or { + "id": jid, + "next_run_at": "2026-08-22T21:00:00+00:00", + "fire_claim": { + "by": "owner", "at": acquired_at, "fire_at": acquired_at, + }, + } + )) + + def create(jid, *, source, **kwargs): + events.append(("create", kwargs.get("fire_identity"))) + return { + "id": "exec-race", "fire_identity": kwargs.get("fire_identity") or "exec-race", + } + + def bind(execution_id, fire_identity): + events.append(("bind", execution_id, fire_identity)) + return {"id": execution_id, "fire_identity": fire_identity} + + monkeypatch.setattr(executions, "create_execution", create) + monkeypatch.setattr(executions, "bind_execution_fire_identity", bind, raising=False) + claimed = InProcessCronScheduler().claim_fire("j1") + + expected = executions.scheduled_fire_identity("j1", acquired_at) + assert events[0] == ("create", None) + assert events[1] == "claim" + assert events[2] == ("bind", "exec-race", expected) + assert claimed["fire_identity"] == expected + + +def test_claim_fire_reuses_immutable_timestamp_identity_for_stale_recovery(monkeypatch): + import cron.executions as executions + import cron.jobs as jobs + from cron.scheduler_provider import InProcessCronScheduler + + created = [] + fire_at = "2026-08-22T19:00:00+00:00" + monkeypatch.setattr(jobs, "get_job", lambda _jid: { + "id": "j1", "next_run_at": "2026-08-22T20:00:00+00:00", + "fire_claim": { + "by": "dead-owner", + "at": "2026-08-22T19:30:00+00:00", + "fire_at": fire_at, + }, + }) + monkeypatch.setattr(jobs, "claim_job_for_fire", lambda jid, **_kwargs: { + "id": jid, "fire_claim": { + "by": "new-owner", + "at": "2026-08-22T20:01:00+00:00", + "fire_at": fire_at, + }, + }) + + def create(jid, *, source, **kwargs): + created.append(kwargs.get("fire_identity")) + return {"id": "exec-2", "fire_identity": kwargs.get("fire_identity")} + + monkeypatch.setattr(executions, "create_execution", create) + claimed = InProcessCronScheduler().claim_fire("j1") + + expected = executions.scheduled_fire_identity("j1", fire_at) + assert created == [expected] + assert claimed["fire_identity"] == expected + + +def test_external_recovery_after_heartbeat_reuses_original_fire_identity( + monkeypatch, tmp_path, +): + from datetime import datetime, timedelta, timezone + + import cron.executions as executions + import cron.jobs as jobs + from cron.scheduler_provider import InProcessCronScheduler + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr( + executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db", + ) + current = [datetime(2026, 8, 22, 19, 0, tzinfo=timezone.utc)] + monkeypatch.setattr(jobs, "_hermes_now", lambda: current[0]) + + job = jobs.create_job(prompt="bounded", schedule="every 5m", name="recovery") + provider = InProcessCronScheduler() + first = provider.claim_fire(job["id"]) + assert first is not None + plan = [{ + "target": {"platform": "telegram", "chat_id": "123"}, + "component": "text", "ordinal": 0, "content": "bounded", + }] + executions.preregister_receipt_plan( + first["execution_id"], + fire_identity=first["fire_identity"], + components=plan, + ) + + current[0] += timedelta(seconds=30) + assert jobs.heartbeat_fire_claim( + job["id"], expected_owner=first["fire_claim"]["by"], + ) is True + current[0] += timedelta(seconds=301) + + recovery = provider.claim_fire(job["id"]) + assert recovery is not None + assert recovery["fire_identity"] == first["fire_identity"] + with __import__("pytest").raises(ValueError, match="already attempted"): + executions.preregister_receipt_plan( + recovery["execution_id"], + fire_identity=recovery["fire_identity"], + components=plan, + ) def test_fire_due_forwards_manual_force_to_store_claim(monkeypatch): + import cron.executions as executions import cron.jobs as jobs import cron.scheduler as sched from cron.scheduler_provider import InProcessCronScheduler claims = [] + events = [] + fire_at = "2026-08-22T19:00:00+00:00" monkeypatch.setattr( jobs, "claim_job_for_fire", lambda jid, **kw: claims.append((jid, kw)) - or {"id": jid, "name": "t", "fire_claim": {"by": "manual-owner"}}, + or {"id": jid, "name": "t", "fire_claim": { + "by": "manual-owner", "at": fire_at, "fire_at": fire_at, + }}, + ) + monkeypatch.setattr( + executions, + "create_execution", + lambda jid, source: {"id": "exec-force"}, + ) + monkeypatch.setattr( + executions, + "bind_execution_fire_identity", + lambda eid, fire: events.append(("bind", eid, fire)) + or {"id": eid, "fire_identity": fire}, + ) + monkeypatch.setattr( + sched, + "run_one_job", + lambda job, **kw: events.append(("run", job["fire_identity"])) or True, ) - monkeypatch.setattr(sched, "run_one_job", lambda job, **kw: True) assert InProcessCronScheduler().fire_due("j1", force=True) is True assert claims == [("j1", {"force": True, "return_job": True})] + expected = executions.scheduled_fire_identity("j1", fire_at) + assert events == [ + ("bind", "exec-force", expected), + ("run", expected), + ] + + +def test_force_claim_without_immutable_timestamp_fails_before_dispatch(monkeypatch): + import cron.executions as executions + import cron.jobs as jobs + import cron.scheduler as sched + from cron.scheduler_provider import InProcessCronScheduler + + monkeypatch.setattr( + jobs, + "claim_job_for_fire", + lambda jid, **kw: {"id": jid, "fire_claim": {"by": "manual-owner"}}, + ) + monkeypatch.setattr( + executions, + "create_execution", + lambda jid, source: {"id": "exec-force"}, + ) + monkeypatch.setattr(executions, "finish_execution", lambda *args, **kwargs: None) + run = Mock() + monkeypatch.setattr(sched, "run_one_job", run) + + with __import__("pytest").raises( + ValueError, match="acquired fire claim has no immutable timestamp", + ): + InProcessCronScheduler().fire_due("j1", force=True) + run.assert_not_called() def test_fire_due_lost_claim_does_not_run(monkeypatch): @@ -460,6 +683,31 @@ def test_fire_due_lost_claim_does_not_run(monkeypatch): assert ran == [] +def test_claim_loser_is_categorized_before_return(monkeypatch): + import cron.executions as executions + import cron.jobs as jobs + from cron.scheduler_provider import InProcessCronScheduler + + finished = [] + monkeypatch.setattr(jobs, "get_job", lambda _jid: None) + monkeypatch.setattr(jobs, "claim_job_for_fire", lambda _jid, **_kwargs: False) + monkeypatch.setattr( + executions, "create_execution", + lambda _jid, **_kwargs: {"id": "exec-loser", "fire_identity": "exec-loser"}, + ) + monkeypatch.setattr( + executions, "finish_execution", + lambda eid, **kwargs: finished.append((eid, kwargs)), + ) + + assert InProcessCronScheduler().claim_fire("j1") is None + assert finished == [("exec-loser", { + "success": False, + "error": "Fire claim was not acquired", + "error_kind": "claim_lost", + })] + + def test_fire_due_missing_job_does_not_run(monkeypatch): """If the job vanished before atomic claim, fire_due does not run it.""" import cron.jobs as jobs diff --git a/tests/cron/test_script_claim_heartbeat.py b/tests/cron/test_script_claim_heartbeat.py index 11fb4501b02c..e654ce731174 100644 --- a/tests/cron/test_script_claim_heartbeat.py +++ b/tests/cron/test_script_claim_heartbeat.py @@ -386,12 +386,26 @@ def _run_job( "name": "reclaimed agent", "prompt": "work", "execution_id": "stale-execution", - "fire_claim": {"at": "2026-07-12T12:00:00+00:00", "by": "stale-owner"}, + "fire_identity": scheduler.scheduled_fire_identity( + "reclaimed-agent", "2026-07-12T12:00:00+00:00", + ), + "fire_claim": { + "at": "2026-07-12T12:00:00+00:00", + "fire_at": "2026-07-12T12:00:00+00:00", + "by": "stale-owner", + }, } monkeypatch.setattr(scheduler, "_RUN_CLAIM_HEARTBEAT_SECONDS", 0.01) monkeypatch.setattr(scheduler, "heartbeat_fire_claim", _heartbeat) monkeypatch.setattr(scheduler, "run_job", _run_job) monkeypatch.setattr(scheduler, "claim_dispatch", lambda job_id: True) + monkeypatch.setattr( + scheduler, + "bind_execution_fire_identity", + lambda execution_id, fire_identity: { + "id": execution_id, "fire_identity": fire_identity, + }, + ) monkeypatch.setattr(scheduler, "mark_execution_running", lambda execution_id: {}) monkeypatch.setattr(scheduler, "finish_execution", lambda *args, **kwargs: None) save_output = MagicMock() @@ -558,12 +572,26 @@ def owned_fence(*_args, **_kwargs): job = { "id": "terminal-cas", "execution_id": "execution-cas", + "fire_identity": scheduler.scheduled_fire_identity( + "terminal-cas", "2026-07-12T12:00:00+00:00", + ), "name": "terminal-cas", - "fire_claim": {"at": "2026-07-12T12:00:00+00:00", "by": "owner"}, + "fire_claim": { + "at": "2026-07-12T12:00:00+00:00", + "fire_at": "2026-07-12T12:00:00+00:00", + "by": "owner", + }, } finish = MagicMock() monkeypatch.setattr(scheduler, "heartbeat_fire_claim", lambda *args, **kwargs: True) monkeypatch.setattr(scheduler, "claim_dispatch", lambda *_args, **_kwargs: True) + monkeypatch.setattr( + scheduler, + "bind_execution_fire_identity", + lambda execution_id, fire_identity: { + "id": execution_id, "fire_identity": fire_identity, + }, + ) monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_args: {}) monkeypatch.setattr( scheduler, diff --git a/tests/cron/test_shutdown_interrupt.py b/tests/cron/test_shutdown_interrupt.py index a8bb3bf28f13..dc41693ac90b 100644 --- a/tests/cron/test_shutdown_interrupt.py +++ b/tests/cron/test_shutdown_interrupt.py @@ -363,7 +363,11 @@ def test_replacement_execution_of_same_job_is_not_poisoned(self): "id": "job-1", "name": "test job", "prompt": "do work", - "fire_claim": {"by": "replacement-owner"}, + "fire_claim": { + "by": "replacement-owner", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }, } with patch("cron.scheduler.claim_dispatch", return_value=True), \ patch("agent.secret_scope.set_secret_scope", return_value=None), \ @@ -432,7 +436,11 @@ def _job(self): "id": "job-be", "name": "base exc", "prompt": "p", - "fire_claim": {"by": "owner-be"}, + "fire_claim": { + "by": "owner-be", + "at": "2026-08-22T19:00:00+00:00", + "fire_at": "2026-08-22T19:00:00+00:00", + }, } def _patches(self, run_side_effect): diff --git a/tests/cron/test_transport_receipt_scheduler.py b/tests/cron/test_transport_receipt_scheduler.py new file mode 100644 index 000000000000..aa324a0ef187 --- /dev/null +++ b/tests/cron/test_transport_receipt_scheduler.py @@ -0,0 +1,845 @@ +"""End-to-end conservative cron transport-receipt integration tests.""" + +from __future__ import annotations + +import asyncio +import hashlib +import sqlite3 +import subprocess +from concurrent.futures import Future +from unittest.mock import AsyncMock, patch + +from cron.scheduler import _deliver_result +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.platforms.base import SendResult, TransportReceipt, TransportTarget + + +class _PlannedTelegramAdapter: + supports_inchannel_continuable = False + + def __init__(self, chunks: list[str]): + self.chunks = chunks + self.send_calls = 0 + self.last_metadata = None + + def plan_transport_text(self, content: str) -> list[str]: + assert content + return list(self.chunks) + + async def send(self, chat_id: str, content: str, metadata=None) -> SendResult: + self.send_calls += 1 + metadata = metadata or {} + self.last_metadata = dict(metadata) + actual_thread = str(metadata.get("thread_id")) if metadata.get("thread_id") else None + requested_identity = metadata.get("_transport_receipt_requested_target") or {} + requested = TransportTarget( + str(requested_identity.get("platform") or "telegram"), + str(requested_identity.get("chat_id") or chat_id), + (str(requested_identity["thread_id"]) if requested_identity.get("thread_id") else None), + ) + actual = TransportTarget("telegram", str(chat_id), actual_thread) + return SendResult( + success=True, + message_id="provider-0", + receipts=tuple( + TransportReceipt( + outcome="delivered", + provider_message_id=f"provider-{ordinal}", + requested_target=requested, + actual_target=actual, + component="text", + ordinal=ordinal, + ) + for ordinal, _chunk in enumerate(self.chunks) + ), + ) + + +class _PartialTelegramAdapter(_PlannedTelegramAdapter): + async def send(self, chat_id: str, content: str, metadata=None) -> SendResult: + self.send_calls += 1 + target = TransportTarget("telegram", str(chat_id)) + return SendResult( + success=False, + error="second chunk failed", + receipts=(TransportReceipt( + outcome="delivered", provider_message_id="provider-0", + requested_target=target, actual_target=target, + component="text", ordinal=0, + ),), + ) + + +class _TypedMediaTelegramAdapter(_PlannedTelegramAdapter): + async def send_document( + self, chat_id: str, file_path: str, metadata=None, + ) -> SendResult: + metadata = metadata or {} + requested_identity = metadata["_transport_receipt_requested_target"] + requested = TransportTarget( + str(requested_identity["platform"]), + str(requested_identity["chat_id"]), + (str(requested_identity["thread_id"]) if requested_identity.get("thread_id") else None), + ) + actual = TransportTarget("telegram", str(chat_id)) + ordinal = metadata["_transport_receipt_ordinal"] + return SendResult( + success=True, + message_id=f"media-provider-{ordinal}", + receipts=(TransportReceipt( + outcome="delivered", + provider_message_id=f"media-provider-{ordinal}", + requested_target=requested, + actual_target=actual, + component="media", + ordinal=ordinal, + ),), + ) + + +class _UnknownMediaTelegramAdapter(_PlannedTelegramAdapter): + async def send_document( + self, chat_id: str, file_path: str, metadata=None, + ) -> SendResult: + metadata = metadata or {} + requested_identity = metadata["_transport_receipt_requested_target"] + requested = TransportTarget( + str(requested_identity["platform"]), + str(requested_identity["chat_id"]), + (str(requested_identity["thread_id"]) if requested_identity.get("thread_id") else None), + ) + ordinal = metadata["_transport_receipt_ordinal"] + return SendResult( + success=False, + error="media delivery outcome is unknown", + error_kind="unknown", + receipts=(TransportReceipt( + outcome="unknown", + requested_target=requested, + component="media", + ordinal=ordinal, + ),), + ) + + +class _FailedMediaWithDeliveredReceiptAdapter(_TypedMediaTelegramAdapter): + async def send_document( + self, chat_id: str, file_path: str, metadata=None, + ) -> SendResult: + delivered = await super().send_document(chat_id, file_path, metadata) + return SendResult( + success=False, + error="provider reported media failure", + receipts=delivered.receipts, + ) + + +def _gateway_config() -> GatewayConfig: + return GatewayConfig( + platforms={Platform.TELEGRAM: PlatformConfig(enabled=True)}, + ) + + +def _running_loop(): + loop = type("Loop", (), {"is_running": lambda self: True})() + return loop + + +def _run_coroutine_threadsafe(coro, _loop): + future = Future() + try: + future.set_result(asyncio.run(coro)) + except BaseException as exc: # noqa: BLE001 - test transports exception faithfully + future.set_exception(exc) + return future + + +def _job() -> dict: + return { + "id": "receipt-e2e", + "name": "receipt-e2e", + "deliver": "origin", + "origin": {"platform": "telegram", "chat_id": "123"}, + } + + +def test_standalone_telegram_preregistration_uses_exact_formatted_chunks(): + from cron.scheduler import _receipt_text_chunks_for_target + from tools.send_message_tool import _plan_standalone_telegram_text + + content = ("- bounded item!\n" * 300).strip() + _formatted, actual_chunks, _has_html, _caption = ( + _plan_standalone_telegram_text(content) + ) + planned_chunks = _receipt_text_chunks_for_target(None, "telegram", content) + + assert len(actual_chunks) > 1 + assert planned_chunks == actual_chunks + assert planned_chunks != [content] + + +def test_scheduler_rejects_hostile_adapter_plan_and_receipt_containers(): + from cron.scheduler import ( + _confirm_adapter_delivery, + _persist_target_text_receipts, + _receipt_text_chunks_for_target, + ) + + class HostileList(list): + def __bool__(self): + raise AssertionError("hostile plan truthiness was evaluated") + + def __iter__(self): + raise AssertionError("hostile plan was iterated") + + class HostileTuple(tuple): + def __bool__(self): + raise AssertionError("hostile receipts truthiness was evaluated") + + def __iter__(self): + raise AssertionError("hostile receipts were iterated") + + class HostileResult: + def __getattribute__(self, _name): + raise AssertionError("hostile adapter result attribute was read") + + descriptor_calls = [] + + class HostileFieldsDescriptor: + @property + def __dict__(self): + descriptor_calls.append("called") + return {"success": True} + + adapter = _PlannedTelegramAdapter(["unused"]) + adapter.plan_transport_text = lambda _content: HostileList(["private"]) + with __import__("pytest").raises(ValueError, match="planner"): + _receipt_text_chunks_for_target( + {Platform.TELEGRAM: adapter}, "telegram", "bounded", + ) + + assert _persist_target_text_receipts( + HostileTuple(()), + {("telegram", "123", "", "text", 0): "attempt"}, + {"platform": "telegram", "chat_id": "123", "thread_id": ""}, + components={"text"}, + ) is False + assert _confirm_adapter_delivery(HostileResult()) is False + assert _confirm_adapter_delivery(HostileFieldsDescriptor()) is False + assert descriptor_calls == [] + + +def test_persisted_non_delivered_receipts_do_not_satisfy_component_plan(): + from cron.scheduler import _persist_target_text_receipts + + target = TransportTarget("telegram", "123") + attempts = {("telegram", "123", "", "media", 0): "attempt"} + requested = {"platform": "telegram", "chat_id": "123", "thread_id": ""} + receipts = ( + TransportReceipt( + outcome="unknown", + requested_target=target, + component="media", + ordinal=0, + ), + TransportReceipt( + outcome="failed", + requested_target=target, + failure_kind="pre_dispatch", + component="media", + ordinal=0, + ), + ) + + for receipt in receipts: + with patch("cron.scheduler.record_transport_receipt", return_value=True): + assert _persist_target_text_receipts( + (receipt,), attempts, requested, components={"media"} + ) is False + + +def test_delivered_receipt_to_different_actual_target_does_not_satisfy_plan(): + from cron.scheduler import _persist_target_text_receipts + + requested_target = TransportTarget("telegram", "123", "topic-7") + actual_target = TransportTarget("telegram", "123") + attempts = {("telegram", "123", "topic-7", "text", 0): "attempt"} + requested = { + "platform": "telegram", + "chat_id": "123", + "thread_id": "topic-7", + } + receipt = TransportReceipt( + outcome="delivered", + requested_target=requested_target, + actual_target=actual_target, + provider_message_id="provider-ack-1", + component="text", + ordinal=0, + ) + + with patch("cron.scheduler.record_transport_receipt", return_value=True): + assert _persist_target_text_receipts( + (receipt,), attempts, requested, components={"text"} + ) is False + + +def test_standalone_telegram_caption_preregistration_omits_unsent_text_component(): + from cron.scheduler import _receipt_text_chunks_for_target + from tools.send_message_tool import _plan_standalone_telegram_text + + media = [("/tmp/bounded-image.jpg", False)] + content = "bounded caption" + _formatted, actual_chunks, _has_html, caption = ( + _plan_standalone_telegram_text(content, media_files=media) + ) + planned_chunks = _receipt_text_chunks_for_target( + None, "telegram", content, media_files=media, + ) + + assert caption is not None + assert actual_chunks == [] + assert planned_chunks == actual_chunks + + +def test_scheduler_uses_standalone_telegram_plan_when_adapter_loop_is_unavailable( + monkeypatch, tmp_path +): + import cron.executions as executions + from gateway.config import Platform + from tools.send_message_tool import _plan_standalone_telegram_text + + monkeypatch.setattr( + executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db" + ) + execution = executions.create_execution("receipt-e2e", source="direct") + content = ("- standalone item!\n" * 300).strip() + expected_chunks = _plan_standalone_telegram_text(content)[1] + adapter = _PlannedTelegramAdapter(["live-adapter-only"]) + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch( + "tools.send_message_tool._send_to_platform", + return_value={"error": "standalone send rejected before provider ack"}, + ), + ): + _deliver_result( + _job(), content, adapters={Platform.TELEGRAM: adapter}, loop=None, + execution_id=execution["id"], fire_identity="fire-no-loop", + ) + + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + hashes = [row[0] for row in conn.execute( + "SELECT content_hash FROM delivery_components ORDER BY ordinal" + )] + assert hashes == [ + hashlib.sha256(chunk.encode("utf-8")).hexdigest() + for chunk in expected_chunks + ] + + +def test_scheduler_preregisters_and_persists_every_planned_text_chunk(monkeypatch, tmp_path): + import cron.executions as executions + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _PlannedTelegramAdapter(["chunk-one", "chunk-two"]) + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + ): + result = _deliver_result( + _job(), "one long logical report", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-chunks", + ) + + assert result is None + assert adapter.send_calls == 1 + assert executions.receipt_summary(execution["id"]) == { + "delivered": 2, + "failed": 0, + "unknown": 0, + "targets_delivered": 1, + } + + +def test_receipt_preregistration_failure_stops_before_adapter_send(monkeypatch): + adapter = _PlannedTelegramAdapter(["chunk-one"]) + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("cron.scheduler.preregister_receipt_plan", side_effect=RuntimeError("private database path")), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + ): + result = _deliver_result( + _job(), "report body", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id="execution-id", fire_identity="fire-preregister-fail", + ) + + assert result == "delivery receipt plan could not be persisted; no delivery was sent" + assert adapter.send_calls == 0 + assert "private database path" not in result + + +def test_scheduler_rejects_target_subclasses_before_magic_methods(monkeypatch): + adapter = _PlannedTelegramAdapter(["chunk-one"]) + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile target truthiness was evaluated") + + def __str__(self): + raise AssertionError("hostile target was stringified") + + def __hash__(self): + raise AssertionError("hostile target was hashed") + + def __eq__(self, _other): + raise AssertionError("hostile target was compared") + + job = _job() + job["origin"] = {"platform": "telegram", "chat_id": HostileText("123")} + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + ): + result = _deliver_result( + job, "report body", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id="execution-id", fire_identity="fire-id", + ) + + assert result == "delivery target is invalid; no delivery was sent" + assert adapter.send_calls == 0 + + +def test_scheduler_rejects_execution_identity_subclasses_before_magic_methods(): + adapter = _PlannedTelegramAdapter(["chunk-one"]) + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile identity truthiness was evaluated") + + def __str__(self): + raise AssertionError("hostile identity was stringified") + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + ): + result = _deliver_result( + _job(), "report body", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=HostileText("execution-id"), fire_identity="fire-id", + ) + + assert result == "delivery receipt identity is invalid; no delivery was sent" + assert adapter.send_calls == 0 + + +def test_partial_chunk_ack_is_retained_without_standalone_retry(monkeypatch, tmp_path): + import cron.executions as executions + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _PartialTelegramAdapter(["chunk-one", "chunk-two"]) + standalone_calls = AsyncMock(return_value={"success": True}) + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + patch("tools.send_message_tool._send_to_platform", new=standalone_calls), + ): + result = _deliver_result( + _job(), "partial report", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-partial-chunks", + ) + + assert result is not None + standalone_calls.assert_not_awaited() + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, "failed": 0, "unknown": 1, "targets_delivered": 0, + } + + +def test_text_ack_with_opaque_media_remains_partial_and_target_unknown(monkeypatch, tmp_path): + import cron.executions as executions + import gateway.platforms.base as base + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + media = tmp_path / "receipt-report.pdf" + media.write_bytes(b"test-pdf") + monkeypatch.setattr(base, "MEDIA_DELIVERY_SAFE_ROOTS", (tmp_path,)) + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _PlannedTelegramAdapter(["text"]) + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + patch("cron.scheduler._send_media_via_adapter", return_value=[]), + ): + result = _deliver_result( + _job(), f"report\nMEDIA:{media}", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-media", + ) + + assert result is not None + assert "media acknowledgement" in result + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, + "failed": 0, + "unknown": 1, + "targets_delivered": 0, + } + + +def test_live_adapter_typed_media_ack_completes_planned_target(monkeypatch, tmp_path): + import cron.executions as executions + import gateway.platforms.base as base + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + media = tmp_path / "typed-receipt-report.pdf" + media.write_bytes(b"test-pdf") + monkeypatch.setattr(base, "MEDIA_DELIVERY_SAFE_ROOTS", (tmp_path,)) + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _TypedMediaTelegramAdapter(["text"]) + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + ): + result = _deliver_result( + _job(), f"report\nMEDIA:{media}", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-typed-media", + ) + + assert result is None + assert executions.receipt_summary(execution["id"]) == { + "delivered": 2, + "failed": 0, + "unknown": 0, + "targets_delivered": 1, + } + + +def test_live_adapter_unknown_media_ack_stays_partial_without_session_side_effects( + monkeypatch, tmp_path +): + import cron.executions as executions + import gateway.platforms.base as base + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + media = tmp_path / "unknown-receipt-report.pdf" + media.write_bytes(b"test-pdf") + monkeypatch.setattr(base, "MEDIA_DELIVERY_SAFE_ROOTS", (tmp_path,)) + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _UnknownMediaTelegramAdapter(["text"]) + job = _job() + job["attach_to_session"] = True + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + patch("cron.scheduler._open_continuable_cron_thread", return_value="created-thread"), + patch("cron.scheduler._seed_cron_thread_session") as seed_thread, + patch("cron.scheduler._maybe_mirror_cron_delivery") as mirror_delivery, + ): + result = _deliver_result( + job, f"report\nMEDIA:{media}", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-unknown-media", + ) + + assert result is not None + assert "media" in result + seed_thread.assert_not_called() + mirror_delivery.assert_not_called() + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, + "failed": 0, + "unknown": 1, + "targets_delivered": 0, + } + + +def test_live_adapter_media_error_prevents_delivery_side_effects_even_with_delivered_receipt( + monkeypatch, tmp_path +): + import cron.executions as executions + import gateway.platforms.base as base + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + media = tmp_path / "failed-media-report.pdf" + media.write_bytes(b"test-pdf") + monkeypatch.setattr(base, "MEDIA_DELIVERY_SAFE_ROOTS", (tmp_path,)) + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _FailedMediaWithDeliveredReceiptAdapter(["text"]) + standalone_calls = AsyncMock(return_value={"success": True}) + job = _job() + job["attach_to_session"] = True + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + patch("cron.scheduler._open_continuable_cron_thread", return_value=None), + patch("cron.scheduler._maybe_mirror_cron_delivery") as mirror_delivery, + patch("tools.send_message_tool._send_to_platform", new=standalone_calls), + ): + result = _deliver_result( + job, f"report\nMEDIA:{media}", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-failed-media", + ) + + assert result is not None + assert "provider reported media failure" in result + mirror_delivery.assert_not_called() + standalone_calls.assert_not_awaited() + assert executions.receipt_summary(execution["id"]) == { + "delivered": 2, + "failed": 0, + "unknown": 0, + "targets_delivered": 1, + } + + +def test_new_continuation_thread_preserves_preregistered_requested_target(monkeypatch, tmp_path): + import cron.executions as executions + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("receipt-e2e", source="direct") + adapter = _PlannedTelegramAdapter(["brief"]) + job = _job() + job["attach_to_session"] = True + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("asyncio.run_coroutine_threadsafe", side_effect=_run_coroutine_threadsafe), + patch("cron.scheduler._open_continuable_cron_thread", return_value="created-thread") as open_thread, + patch("cron.scheduler._seed_cron_thread_session"), + patch("cron.scheduler._maybe_mirror_cron_delivery"), + ): + result = _deliver_result( + job, "continuable brief", + adapters={Platform.TELEGRAM: adapter}, loop=_running_loop(), + execution_id=execution["id"], fire_identity="fire-thread", + ) + + assert result is None + open_thread.assert_called_once() + assert adapter.last_metadata["_transport_receipt_requested_target"] == { + "platform": "telegram", "chat_id": "123", "thread_id": "", + } + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, + "failed": 0, + "unknown": 0, + "targets_delivered": 0, + } + + +def test_standalone_typed_receipt_is_persisted(monkeypatch, tmp_path): + import cron.executions as executions + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("receipt-e2e", source="direct") + target = TransportTarget("telegram", "123") + standalone_result = { + "success": True, + "message_id": "standalone-1", + "receipts": ( + TransportReceipt( + outcome="delivered", provider_message_id="standalone-1", + requested_target=target, actual_target=target, + component="text", ordinal=0, + ), + ), + } + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("tools.send_message_tool._send_to_platform", return_value=standalone_result), + ): + result = _deliver_result( + _job(), "standalone report", adapters=None, loop=None, + execution_id=execution["id"], fire_identity="fire-standalone-typed", + ) + + assert result is None + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, "failed": 0, "unknown": 0, "targets_delivered": 1, + } + + +def test_standalone_legacy_success_remains_unknown(monkeypatch, tmp_path): + import cron.executions as executions + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("receipt-e2e", source="direct") + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("tools.send_message_tool._send_to_platform", return_value={"success": True, "message_id": "legacy"}), + ): + result = _deliver_result( + _job(), "standalone report", adapters=None, loop=None, + execution_id=execution["id"], fire_identity="fire-standalone-legacy", + ) + + assert result is not None + assert "typed receipt" in result + assert executions.receipt_summary(execution["id"]) == { + "delivered": 0, "failed": 0, "unknown": 1, "targets_delivered": 0, + } + + +def test_bot_chat_exit_zero_persists_observed_unknown_for_exact_query_bytes( + monkeypatch, tmp_path +): + import cron.executions as executions + import cron.scheduler as scheduler + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("bot-receipt", source="direct") + captured = {} + + def fake_run(argv, **_kwargs): + query_file = argv[argv.index("--query-file") + 1] + with open(query_file, encoding="utf-8") as handle: + captured["message"] = handle.read() + return subprocess.CompletedProcess(argv, 0, "", "") + + job = {"id": "bot-receipt", "name": "bot receipt", "deliver": "bot-chat:research"} + with ( + patch("gateway.config.load_gateway_config", return_value=GatewayConfig()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("cron.scheduler._resolve_delivery_targets", return_value=[{ + "platform": "bot-chat", "chat_id": "research", "thread_id": None, + }]), + patch.object(scheduler.shutil, "which", return_value="/usr/bin/hermes"), + patch.object(scheduler.subprocess, "run", side_effect=fake_run), + ): + result = _deliver_result( + job, "bot payload", adapters=None, loop=None, + execution_id=execution["id"], fire_identity="fire-bot-chat", + ) + + assert result == "bot-chat delivery confirmation unavailable" + assert executions.receipt_summary(execution["id"]) == { + "delivered": 0, "failed": 0, "unknown": 1, "targets_delivered": 0, + } + with sqlite3.connect(executions.EXECUTIONS_FILE) as conn: + row = conn.execute( + """SELECT c.component, c.ordinal, c.content_hash, + a.outcome, a.observed_at + FROM delivery_attempts a + JOIN delivery_components c ON c.id=a.component_id""" + ).fetchone() + count = conn.execute("SELECT COUNT(*) FROM delivery_components").fetchone()[0] + assert count == 1 + assert row == ( + "text", 0, + hashlib.sha256(captured["message"].encode("utf-8")).hexdigest(), + "unknown", row[4], + ) + assert row[4] is not None + + +def test_standalone_text_ack_with_opaque_media_remains_partial(monkeypatch, tmp_path): + import cron.executions as executions + import gateway.platforms.base as base + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + media = tmp_path / "standalone-report.pdf" + media.write_bytes(b"test-pdf") + monkeypatch.setattr(base, "MEDIA_DELIVERY_SAFE_ROOTS", (tmp_path,)) + execution = executions.create_execution("receipt-e2e", source="direct") + target = TransportTarget("telegram", "123") + standalone_result = { + "success": True, + "receipts": (TransportReceipt( + outcome="delivered", provider_message_id="standalone-text", + requested_target=target, actual_target=target, + component="text", ordinal=0, + ),), + } + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("tools.send_message_tool._send_to_platform", return_value=standalone_result), + ): + result = _deliver_result( + _job(), f"{'report ' + ('x' * 1100)}\nMEDIA:{media}", + adapters=None, loop=None, + execution_id=execution["id"], fire_identity="fire-standalone-media", + ) + + assert result is not None + assert "media acknowledgement" in result + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, "failed": 0, "unknown": 1, "targets_delivered": 0, + } + + +def test_standalone_typed_text_and_media_acks_complete_target(monkeypatch, tmp_path): + import cron.executions as executions + import gateway.platforms.base as base + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + media = tmp_path / "standalone-confirmed.pdf" + media.write_bytes(b"test-pdf") + monkeypatch.setattr(base, "MEDIA_DELIVERY_SAFE_ROOTS", (tmp_path,)) + execution = executions.create_execution("receipt-e2e", source="direct") + target = TransportTarget("telegram", "123") + standalone_result = { + "success": True, + "receipts": ( + TransportReceipt( + outcome="delivered", provider_message_id="standalone-text", + requested_target=target, actual_target=target, + component="text", ordinal=0, + ), + TransportReceipt( + outcome="delivered", provider_message_id="standalone-media", + requested_target=target, actual_target=target, + component="media", ordinal=0, + ), + ), + } + + with ( + patch("gateway.config.load_gateway_config", return_value=_gateway_config()), + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), + patch("tools.send_message_tool._send_to_platform", return_value=standalone_result), + ): + result = _deliver_result( + _job(), f"{'report ' + ('x' * 1100)}\nMEDIA:{media}", + adapters=None, loop=None, + execution_id=execution["id"], fire_identity="fire-standalone-confirmed-media", + ) + + assert result is None + assert executions.receipt_summary(execution["id"]) == { + "delivered": 2, "failed": 0, "unknown": 0, "targets_delivered": 1, + } diff --git a/tests/gateway/test_delivery.py b/tests/gateway/test_delivery.py index c8991fd5d158..6ac1c04b6197 100644 --- a/tests/gateway/test_delivery.py +++ b/tests/gateway/test_delivery.py @@ -21,6 +21,338 @@ def test_explicit_telegram_chat(self): assert target.is_explicit is True +class TestTransportReceiptContract: + def test_transport_target_rejects_string_subclasses_without_magic_methods(self): + from gateway.platforms.base import TransportTarget + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile text truthiness was evaluated") + + def __len__(self): + raise AssertionError("hostile text length was evaluated") + + def __eq__(self, _other): + raise AssertionError("hostile text was compared") + + def __hash__(self): + raise AssertionError("hostile text was hashed") + + with pytest.raises(ValueError, match="platform must be"): + TransportTarget(platform=HostileText("telegram"), chat_id="123") + + def test_transport_receipt_rejects_subclasses_before_magic_methods(self): + from gateway.platforms.base import TransportReceipt, TransportTarget + + target = TransportTarget(platform="telegram", chat_id="123") + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile text truthiness was evaluated") + + def __hash__(self): + raise AssertionError("hostile text was hashed") + + def __eq__(self, _other): + raise AssertionError("hostile text was compared") + + with pytest.raises(ValueError, match="outcome must be"): + TransportReceipt(outcome=HostileText("unknown"), requested_target=target) + + class HostileTarget(TransportTarget): + pass + + with pytest.raises(TypeError, match="requested_target"): + TransportReceipt( + outcome="unknown", + requested_target=HostileTarget("telegram", "123"), + ) + + def test_transport_receipt_rejects_datetime_subclasses_and_custom_tzinfo(self): + from datetime import datetime, timedelta, timezone, tzinfo + from gateway.platforms.base import TransportReceipt, TransportTarget + + target = TransportTarget(platform="telegram", chat_id="123") + + class HostileDateTime(datetime): + def astimezone(self, *_args, **_kwargs): + raise AssertionError("hostile datetime conversion was called") + + hostile_datetime = HostileDateTime.now(timezone.utc) + with pytest.raises(ValueError, match="timezone-aware"): + TransportReceipt( + outcome="unknown", requested_target=target, + observed_at=hostile_datetime, + ) + + class HostileTimezone(tzinfo): + def utcoffset(self, _dt): + raise AssertionError("hostile timezone offset was called") + + def dst(self, _dt): + return timedelta(0) + + custom_tz_datetime = datetime(2026, 8, 23, tzinfo=HostileTimezone()) + with pytest.raises(ValueError, match="timezone-aware"): + TransportReceipt( + outcome="unknown", requested_target=target, + observed_at=custom_tz_datetime, + ) + + def test_matrix_receipt_target_metadata_rejects_subclasses_without_methods(self): + from plugins.platforms.matrix.adapter import MatrixAdapter + + class HostileDict(dict): + def __bool__(self): + raise AssertionError("hostile metadata truthiness was evaluated") + + def get(self, *_args, **_kwargs): + raise AssertionError("hostile metadata get was called") + + with pytest.raises(TypeError, match="metadata"): + MatrixAdapter._transport_receipt_targets( + "!room:example.org", HostileDict({}) + ) + with pytest.raises(TypeError, match="requested target"): + MatrixAdapter._transport_receipt_targets( + "!room:example.org", + {"_transport_receipt_requested_target": HostileDict({})}, + ) + + @pytest.mark.asyncio + async def test_matrix_send_rejects_receipt_metadata_before_provider_dispatch(self): + from types import SimpleNamespace + from unittest.mock import AsyncMock + from plugins.platforms.matrix.adapter import MatrixAdapter + + adapter = object.__new__(MatrixAdapter) + adapter.plan_transport_text = lambda _content: ["bounded"] + adapter._client = SimpleNamespace(send_message_event=AsyncMock()) + + class HostileDict(dict): + def __bool__(self): + raise AssertionError("hostile metadata truthiness was evaluated") + + def get(self, *_args, **_kwargs): + raise AssertionError("hostile metadata get was called") + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile content truthiness was evaluated") + + def __str__(self): + raise AssertionError("hostile content was stringified") + + metadata_result = await adapter.send( + "!room:example.org", "bounded", metadata=HostileDict({}) + ) + content_result = await adapter.send( + "!room:example.org", HostileText("bounded"), metadata={} + ) + + for result in (metadata_result, content_result): + assert result.success is False + assert result.error_kind == "invalid_transport_receipt" + adapter._client.send_message_event.assert_not_awaited() + + media_adapter = object.__new__(MatrixAdapter) + media_adapter._client = SimpleNamespace( + upload_media=AsyncMock(), send_message_event=AsyncMock(), + ) + media_adapter._max_media_bytes = 1024 + media_adapter._encryption = False + media_result = await media_adapter._upload_and_send( + "!room:example.org", b"bounded", "report.pdf", + "application/pdf", "m.file", metadata=HostileDict({}), + ) + assert media_result.success is False + assert media_result.error_kind == "invalid_transport_receipt" + media_adapter._client.upload_media.assert_not_awaited() + media_adapter._client.send_message_event.assert_not_awaited() + + def test_provider_message_id_normalization_never_calls_hostile_stringification(self): + from gateway.platforms.base import normalize_transport_provider_message_id + + class HostileText(str): + def __str__(self): + raise AssertionError("hostile provider id was stringified") + + def __len__(self): + raise AssertionError("hostile provider id length was evaluated") + + class HostileTextMeta(type): + def __getattribute__(self, _name): + raise AssertionError("hostile provider id type metadata was read") + + class HostileMetaText(str, metaclass=HostileTextMeta): + pass + + class HostileObject: + def __str__(self): + raise AssertionError("provider object was stringified") + + class_spoof_calls = [] + + class HostileClassSpoof: + @property + def __class__(self): + class_spoof_calls.append("called") + return str + + assert normalize_transport_provider_message_id(HostileText("provider-1")) == "provider-1" + assert normalize_transport_provider_message_id(HostileMetaText("provider-2")) == "provider-2" + assert normalize_transport_provider_message_id(42) == "42" + assert normalize_transport_provider_message_id(HostileObject()) is None + assert normalize_transport_provider_message_id(HostileClassSpoof()) is None + assert class_spoof_calls == [] + assert normalize_transport_provider_message_id(True) is None + + def test_receipt_targets_are_typed_and_observation_is_always_aware(self): + from datetime import datetime + from gateway.platforms.base import TransportReceipt, TransportTarget + + target = TransportTarget(platform="telegram", chat_id="123") + receipt = TransportReceipt( + outcome="delivered", provider_message_id="1", + requested_target=target, actual_target=target, + ) + assert receipt.observed_at is not None + assert receipt.observed_at.utcoffset() is not None + for requested, actual in (({}, target), (target, object())): + with pytest.raises(TypeError, match="TransportTarget"): + TransportReceipt( + outcome="delivered", provider_message_id="1", + requested_target=requested, actual_target=actual, + ) + with pytest.raises(ValueError, match="timezone-aware"): + TransportReceipt( + outcome="delivered", provider_message_id="1", + requested_target=target, actual_target=target, + observed_at=datetime.now(), + ) + from datetime import timedelta, timezone + + for unbounded in ( + datetime(1969, 12, 31, tzinfo=timezone.utc), + datetime.now(timezone.utc) + timedelta(minutes=6), + ): + with pytest.raises(ValueError, match="bounded"): + TransportReceipt( + outcome="delivered", provider_message_id="1", + requested_target=target, actual_target=target, + observed_at=unbounded, + ) + with pytest.raises(ValueError, match="ordinal"): + TransportReceipt( + outcome="delivered", provider_message_id="1", + requested_target=target, actual_target=target, ordinal=True, + ) + + def test_receipts_are_ordered_component_evidence_not_a_final_id_alias(self): + from gateway.platforms.base import TransportReceipt, TransportTarget + + target = TransportTarget(platform="telegram", chat_id="123") + first = TransportReceipt( + outcome="delivered", provider_message_id="1", requested_target=target, + actual_target=target, component="text", ordinal=0, + ) + second = TransportReceipt( + outcome="delivered", provider_message_id="2", requested_target=target, + actual_target=target, component="text", ordinal=1, + ) + result = SendResult(success=True, message_id="2", receipts=(first, second)) + + assert result.receipt == first + assert result.receipts == (first, second) + with pytest.raises(ValueError, match="ordered"): + SendResult(success=True, receipts=(second, first)) + + def test_send_result_rejects_receipt_subclasses_and_mutations_before_methods(self): + from gateway.platforms.base import TransportReceipt, TransportTarget + + target = TransportTarget("telegram", "123") + receipt = TransportReceipt(outcome="unknown", requested_target=target) + + class HostileTuple(tuple): + def __iter__(self): + raise AssertionError("hostile receipt tuple was iterated") + + def __len__(self): + raise AssertionError("hostile receipt tuple length was evaluated") + + with pytest.raises(ValueError, match="immutable tuple"): + SendResult(success=False, receipts=HostileTuple((receipt,))) + + class ReceiptSubclass(TransportReceipt): + pass + + subclass_receipt = ReceiptSubclass( + outcome="unknown", requested_target=target, + ) + with pytest.raises(ValueError, match="TransportReceipt"): + SendResult(success=False, receipts=(subclass_receipt,)) + + class HostileText(str): + def __hash__(self): + raise AssertionError("hostile receipt outcome was hashed") + + def __eq__(self, _other): + raise AssertionError("hostile receipt outcome was compared") + + object.__setattr__(receipt, "outcome", HostileText("unknown")) + with pytest.raises(ValueError, match="outcome"): + SendResult(success=False, receipts=(receipt,)) + + @pytest.mark.parametrize("bad", [" id", "id ", "id\nnext", "id\u00a0next"]) + def test_transport_ids_reject_controls_and_whitespace_confusables(self, bad): + from gateway.platforms.base import TransportTarget + + with pytest.raises(ValueError): + TransportTarget(platform="telegram", chat_id=bad) + + def test_delivered_receipt_requires_provider_evidence_and_actual_target(self): + from gateway.platforms.base import TransportReceipt, TransportTarget + + requested = TransportTarget(platform="matrix", chat_id="!requested:example.org") + actual = TransportTarget(platform="matrix", chat_id="!actual:example.org") + receipt = TransportReceipt( + outcome="delivered", + provider_message_id="$event", + requested_target=requested, + actual_target=actual, + ) + + result = SendResult(success=True, message_id="legacy-id", receipt=receipt) + assert result.success is True # legacy semantics stay independent + assert result.message_id == "legacy-id" + assert result.receipt == receipt + + with pytest.raises(ValueError, match="provider_message_id"): + TransportReceipt( + outcome="delivered", + requested_target=requested, + actual_target=actual, + ) + with pytest.raises(ValueError, match="actual_target"): + TransportReceipt( + outcome="delivered", + provider_message_id="$event", + requested_target=requested, + ) + + def test_failed_receipt_requires_bounded_category_and_unknown_is_default(self): + from gateway.platforms.base import TransportReceipt, TransportTarget + + target = TransportTarget(platform="telegram", chat_id="123", thread_id="7") + assert SendResult(success=True).receipt is None + with pytest.raises(ValueError, match="failure_kind"): + TransportReceipt(outcome="failed", requested_target=target) + failed = TransportReceipt( + outcome="failed", requested_target=target, failure_kind="not_configured" + ) + assert failed.failure_kind == "not_configured" + + def test_origin_with_source(self): origin = SessionSource(platform=Platform.TELEGRAM, chat_id="789", thread_id="42") target = DeliveryTarget.parse("origin", origin=origin) diff --git a/tests/gateway/test_matrix.py b/tests/gateway/test_matrix.py index 4c02c9385b59..c7357a888eed 100644 --- a/tests/gateway/test_matrix.py +++ b/tests/gateway/test_matrix.py @@ -320,6 +320,119 @@ def _make_adapter(): return adapter +@pytest.mark.asyncio +async def test_matrix_send_ack_emits_exact_transport_receipt(): + """A Matrix event id is provider evidence, not just legacy message_id.""" + adapter = _make_adapter() + adapter._client = MagicMock() + adapter._client.send_message_event = AsyncMock(return_value="$event-123") + + result = await adapter.send("!room:example.org", "hello") + + assert result.success is True + assert result.receipt is not None + assert result.receipt.outcome == "delivered" + assert result.receipt.provider_message_id == "$event-123" + assert result.receipt.requested_target.chat_id == "!room:example.org" + assert result.receipt.actual_target.chat_id == "!room:example.org" + + +@pytest.mark.asyncio +async def test_matrix_threaded_ack_preserves_requested_and_actual_transport_targets(): + adapter = _make_adapter() + adapter._client = MagicMock() + adapter._client.send_message_event = AsyncMock(return_value="$thread-event") + + result = await adapter.send( + "!room:example.org", + "threaded", + metadata={ + "thread_id": "$actual-root", + "_transport_receipt_requested_target": { + "platform": "matrix", + "chat_id": "!room:example.org", + "thread_id": "$requested-root", + }, + }, + ) + + receipt = result.receipt + assert receipt is not None + assert receipt.requested_target.thread_id == "$requested-root" + assert receipt.actual_target is not None + assert receipt.actual_target.thread_id == "$actual-root" + sent_call = adapter._client.send_message_event.await_args + assert sent_call is not None + sent = sent_call.args[2] + assert sent["m.relates_to"]["event_id"] == "$actual-root" + + +@pytest.mark.asyncio +async def test_matrix_threaded_ack_persists_against_preregistered_target( + monkeypatch, tmp_path, +): + import cron.executions as executions + + monkeypatch.setattr( + executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db", + ) + execution = executions.create_execution("matrix-thread", source="direct") + attempts = executions.preregister_receipt_plan( + execution["id"], + fire_identity="matrix-thread-fire", + components=[{ + "target": { + "platform": "matrix", + "chat_id": "!room:example.org", + "thread_id": "$requested-root", + }, + "component": "text", + "ordinal": 0, + "content": "threaded", + }], + ) + adapter = _make_adapter() + adapter._client = MagicMock() + adapter._client.send_message_event = AsyncMock(return_value="$thread-event") + + result = await adapter.send( + "!room:example.org", + "threaded", + metadata={ + "thread_id": "$actual-root", + "_transport_receipt_requested_target": { + "platform": "matrix", + "chat_id": "!room:example.org", + "thread_id": "$requested-root", + }, + }, + ) + + assert result.receipt is not None + assert executions.record_transport_receipt(attempts[0]["id"], result.receipt) + assert executions.receipt_summary(execution["id"]) == { + "delivered": 1, + "failed": 0, + "unknown": 0, + "targets_delivered": 0, + } + + +@pytest.mark.asyncio +async def test_matrix_multi_chunk_send_emits_one_ordered_receipt_per_ack(): + adapter = _make_adapter() + adapter._client = MagicMock() + adapter._client.send_message_event = AsyncMock(side_effect=["$one", "$two"]) + adapter.truncate_message = MagicMock(return_value=["first", "second"]) + + result = await adapter.send("!room:example.org", "long message") + + assert [receipt.provider_message_id for receipt in result.receipts] == ["$one", "$two"] + assert [(receipt.component, receipt.ordinal) for receipt in result.receipts] == [ + ("text", 0), ("text", 1), + ] + + # --------------------------------------------------------------------------- # Typing indicator # --------------------------------------------------------------------------- @@ -1468,15 +1581,71 @@ async def test_media_preserves_caption_and_thread(self): "image/png", "m.image", caption="Chart caption", - metadata={"thread_id": "$root"}, + metadata={ + "thread_id": "$actual-root", + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 7, + "_transport_receipt_requested_target": { + "platform": "matrix", + "chat_id": "!room:example.org", + "thread_id": "$requested-root", + }, + }, ) assert result.success is True sent = mock_client.send_message_event.await_args.args[2] assert sent["body"] == "Chart caption" assert sent["m.relates_to"]["rel_type"] == "m.thread" - assert sent["m.relates_to"]["event_id"] == "$root" - assert sent["m.relates_to"]["m.in_reply_to"] == {"event_id": "$root"} + assert sent["m.relates_to"]["event_id"] == "$actual-root" + assert sent["m.relates_to"]["m.in_reply_to"] == {"event_id": "$actual-root"} + receipt = result.receipt + assert receipt is not None + assert receipt.provider_message_id == "$event" + assert receipt.component == "media" + assert receipt.ordinal == 7 + assert receipt.requested_target.thread_id == "$requested-root" + assert receipt.actual_target is not None + assert receipt.actual_target.thread_id == "$actual-root" + + @pytest.mark.asyncio + async def test_media_timeout_after_send_is_typed_unknown(self): + adapter = _make_adapter() + mock_client = MagicMock() + mock_client.upload_media = AsyncMock(return_value="mxc://example.org/plain") + mock_client.send_message_event = AsyncMock(side_effect=asyncio.TimeoutError()) + adapter._client = mock_client + + result = await adapter._upload_and_send( + "!room:example.org", + b"image", + "chart.png", + "image/png", + "m.image", + metadata={ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 7, + "_transport_receipt_requested_target": { + "platform": "matrix", + "chat_id": "!room:example.org", + "thread_id": "$requested-root", + }, + }, + ) + + assert result.success is False + assert result.retryable is False + assert result.error_kind == "unknown" + assert result.receipt is not None + assert result.receipt.outcome == "unknown" + assert result.receipt.component == "media" + assert result.receipt.ordinal == 7 + assert result.receipt.provider_message_id is None + assert result.receipt.actual_target is None + assert result.receipt.requested_target.thread_id == "$requested-root" + assert result.receipts == (result.receipt,) + assert mock_client.upload_media.await_count == 1 + assert mock_client.send_message_event.await_count == 1 class TestMatrixDiagnostics: @@ -1593,16 +1762,13 @@ def test_matrix_diagnostics_redacts_recovery_key(self, monkeypatch): class TestMatrixEncryptedSendFallback: @pytest.mark.asyncio - async def test_send_retries_after_e2ee_error(self): - """send() should retry with crypto.share_keys() on E2EE errors.""" + async def test_send_timeout_is_unknown_without_e2ee_retry(self): + """An ambiguous Matrix timeout must never send the same chunk twice.""" adapter = _make_adapter() adapter._encryption = True fake_client = MagicMock() - fake_client.send_message_event = AsyncMock(side_effect=[ - Exception("encryption error"), - "$event123", # mautrix returns EventID string directly - ]) + fake_client.send_message_event = AsyncMock(side_effect=asyncio.TimeoutError()) mock_crypto = MagicMock() mock_crypto.share_keys = AsyncMock() fake_client.crypto = mock_crypto @@ -1610,10 +1776,15 @@ async def test_send_retries_after_e2ee_error(self): result = await adapter.send("!room:example.org", "hello") - assert result.success is True - assert result.message_id == "$event123" - mock_crypto.share_keys.assert_awaited_once() - assert fake_client.send_message_event.await_count == 2 + assert result.success is False + assert result.retryable is False + assert result.error_kind == "unknown" + assert result.receipt is not None + assert result.receipt.outcome == "unknown" + assert result.receipt.provider_message_id is None + assert result.receipt.actual_target is None + mock_crypto.share_keys.assert_not_awaited() + assert fake_client.send_message_event.await_count == 1 # --------------------------------------------------------------------------- diff --git a/tests/gateway/test_telegram_format.py b/tests/gateway/test_telegram_format.py index 2195179b28f2..53a60fb92d83 100644 --- a/tests/gateway/test_telegram_format.py +++ b/tests/gateway/test_telegram_format.py @@ -462,6 +462,77 @@ async def _fake_send_message(**kwargs): assert re.search(r" \\\([0-9]+/[0-9]+\\\)$", sent_texts[-1]) +@pytest.mark.asyncio +async def test_receipt_bound_send_does_not_resend_unplanned_plaintext(adapter): + from telegram.error import BadRequest + + adapter._bot = MagicMock() + adapter._bot.send_message = AsyncMock( + side_effect=[ + BadRequest("Bad Request: can't parse entities"), + SimpleNamespace(message_id=2), + ] + ) + content = "**planned** content" + planned = adapter.plan_transport_text(content) + + result = await adapter.send( + "123", + content, + metadata={ + "_transport_receipt_requested_target": { + "platform": "telegram", + "chat_id": "123", + }, + }, + ) + + assert adapter._bot.send_message.await_count == 1 + assert adapter._bot.send_message.await_args.kwargs["text"] == planned[0] + assert result.success is False + assert result.retryable is False + assert result.error_kind == "provider_rejected" + assert result.receipts == () + + +@pytest.mark.asyncio +async def test_receipt_bound_parse_reject_preserves_prior_chunk_ack(adapter): + from telegram.error import BadRequest + + adapter.MAX_MESSAGE_LENGTH = 80 + adapter._bot = MagicMock() + content = ("**bold** chunk content " * 12).strip() + planned = adapter.plan_transport_text(content) + assert len(planned) > 1 + adapter._bot.send_message = AsyncMock( + side_effect=[ + SimpleNamespace(message_id=1), + BadRequest("Bad Request: can't parse entities"), + SimpleNamespace(message_id=3), + ] + ) + + result = await adapter.send( + "123", + content, + metadata={ + "_transport_receipt_requested_target": { + "platform": "telegram", + "chat_id": "123", + }, + }, + ) + + assert adapter._bot.send_message.await_count == 2 + assert [call.kwargs["text"] for call in adapter._bot.send_message.await_args_list] == planned[:2] + assert result.success is False + assert result.retryable is False + assert result.error_kind == "provider_rejected" + assert len(result.receipts) == 1 + assert result.receipts[0].outcome == "delivered" + assert result.receipts[0].provider_message_id == "1" + + # ========================================================================= # edit_message — streaming Markdown safety # ========================================================================= diff --git a/tests/gateway/test_telegram_thread_fallback.py b/tests/gateway/test_telegram_thread_fallback.py index 046479c2a70d..1ac4cc0705e8 100644 --- a/tests/gateway/test_telegram_thread_fallback.py +++ b/tests/gateway/test_telegram_thread_fallback.py @@ -12,15 +12,17 @@ import socket import types from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from gateway.config import PlatformConfig, Platform from gateway.platforms.base import ( + BasePlatformAdapter, MessageEvent, MessageType, SendResult, + TransportTarget, _reply_anchor_for_event, _thread_metadata_for_source, ) @@ -254,6 +256,441 @@ async def mock_send_message(**kwargs): assert "direct_messages_topic_id" not in call_log[0] +@pytest.mark.asyncio +async def test_send_thread_fallback_receipt_keeps_requested_and_actual_targets(): + """A fallback acknowledgement must not claim the requested topic received it.""" + adapter = _make_adapter() + + async def mock_send_message(**kwargs): + if kwargs.get("message_thread_id") is not None: + raise FakeBadRequest("Message thread not found") + return SimpleNamespace(message_id=270454) + + adapter._bot = SimpleNamespace(send_message=mock_send_message) + result = await adapter.send( + chat_id="-100123", + content="cron topic delivery", + metadata={"thread_id": "270453"}, + ) + + assert result.receipt is not None + assert result.receipt.outcome == "delivered" + assert result.receipt.provider_message_id == "270454" + assert result.receipt.requested_target.thread_id == "270453" + assert result.receipt.actual_target.thread_id is None + + +@pytest.mark.asyncio +async def test_direct_messages_topic_receipt_preserves_logical_topic_target(): + adapter = _make_adapter() + calls = [] + + async def mock_send_message(**kwargs): + calls.append(dict(kwargs)) + return SimpleNamespace(message_id=270455) + + adapter._bot = SimpleNamespace(send_message=mock_send_message) + result = await adapter.send( + chat_id="775566675", + content="cron direct-messages topic delivery", + metadata={ + "direct_messages_topic_id": "270453", + "_transport_receipt_requested_target": { + "platform": "telegram", + "chat_id": "775566675", + "thread_id": "270453", + }, + }, + ) + + assert calls[0]["direct_messages_topic_id"] == 270453 + assert calls[0].get("message_thread_id") is None + assert result.receipt is not None + assert result.receipt.requested_target.thread_id == "270453" + assert result.receipt.actual_target.thread_id == "270453" + + +@pytest.mark.asyncio +async def test_multi_chunk_send_emits_every_telegram_ack_in_order(): + adapter = _make_adapter() + adapter.truncate_message = MagicMock(return_value=["one", "two"]) + adapter._bot = SimpleNamespace( + send_message=AsyncMock(side_effect=[ + SimpleNamespace(message_id=1), SimpleNamespace(message_id=2), + ]) + ) + + result = await adapter.send(chat_id="-100123", content="long message") + + assert [receipt.provider_message_id for receipt in result.receipts] == ["1", "2"] + assert [(receipt.component, receipt.ordinal) for receipt in result.receipts] == [ + ("text", 0), ("text", 1), + ] + + +@pytest.mark.asyncio +async def test_multi_chunk_failure_preserves_earlier_telegram_ack(): + adapter = _make_adapter() + adapter.truncate_message = MagicMock(return_value=["one", "two"]) + adapter._bot = SimpleNamespace( + send_message=AsyncMock(side_effect=[ + SimpleNamespace(message_id=1), RuntimeError("second chunk failed"), + ]) + ) + + result = await adapter.send(chat_id="-100123", content="long message") + + assert result.success is False + assert [receipt.provider_message_id for receipt in result.receipts] == ["1"] + assert [(receipt.component, receipt.ordinal) for receipt in result.receipts] == [ + ("text", 0), + ] + + +@pytest.mark.asyncio +async def test_telegram_native_document_ack_emits_exact_media_receipt(tmp_path): + adapter = _make_adapter() + adapter._bot = SimpleNamespace( + send_document=AsyncMock(return_value=SimpleNamespace(message_id=42)), + ) + document = tmp_path / "report.pdf" + document.write_bytes(b"bounded-document") + + result = await adapter.send_document( + chat_id="-100123", + file_path=str(document), + metadata={ + "thread_id": "270454", + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 7, + "_transport_receipt_requested_target": { + "platform": "telegram", + "chat_id": "-100123", + "thread_id": "270453", + }, + }, + ) + + assert result.success is True + assert result.receipt is not None + assert result.receipt.outcome == "delivered" + assert result.receipt.provider_message_id == "42" + assert (result.receipt.component, result.receipt.ordinal) == ("media", 7) + assert result.receipt.requested_target.thread_id == "270453" + assert result.receipt.actual_target is not None + assert result.receipt.actual_target.thread_id == "270454" + + +@pytest.mark.asyncio +async def test_telegram_native_media_legacy_ack_is_not_upgraded(tmp_path): + adapter = _make_adapter() + adapter._bot = SimpleNamespace( + send_document=AsyncMock(return_value=SimpleNamespace(message_id=43)), + ) + document = tmp_path / "legacy.pdf" + document.write_bytes(b"bounded-document") + + result = await adapter.send_document( + chat_id="-100123", + file_path=str(document), + ) + + assert result.success is True + assert result.message_id == "43" + assert result.receipt is None + assert result.receipts == () + + +@pytest.mark.asyncio +async def test_telegram_invalid_media_receipt_metadata_sends_nothing( + tmp_path, monkeypatch, +): + adapter = _make_adapter() + adapter._bot = SimpleNamespace(send_document=AsyncMock()) + fallback = AsyncMock(return_value=SendResult(success=True, message_id="fallback")) + monkeypatch.setattr(BasePlatformAdapter, "send_document", fallback) + document = tmp_path / "report.pdf" + document.write_bytes(b"bounded-document") + + result = await adapter.send_document( + chat_id="-100123", + file_path=str(document), + metadata={ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": True, + }, + ) + + assert result.success is False + assert result.error == "Invalid transport receipt metadata" + assert result.error_kind == "invalid_transport_receipt" + adapter._bot.send_document.assert_not_awaited() + fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_telegram_media_receipt_metadata_rejects_subclasses_before_fallback( + tmp_path, monkeypatch, +): + adapter = _make_adapter() + adapter._bot = SimpleNamespace(send_document=AsyncMock()) + fallback = AsyncMock(return_value=SendResult(success=True, message_id="fallback")) + monkeypatch.setattr(BasePlatformAdapter, "send_document", fallback) + document = tmp_path / "hostile.pdf" + document.write_bytes(b"bounded-document") + + class HostileDict(dict): + def __bool__(self): + raise AssertionError("hostile metadata truthiness was evaluated") + + def get(self, *_args, **_kwargs): + raise AssertionError("hostile metadata get was called") + + def __contains__(self, _key): + raise AssertionError("hostile metadata membership was evaluated") + + metadata_values = ( + HostileDict({ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 0, + }), + { + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 0, + "_transport_receipt_requested_target": HostileDict({}), + }, + ) + for metadata in metadata_values: + result = await adapter.send_document( + chat_id="-100123", file_path=str(document), metadata=metadata, + ) + assert result.success is False + assert result.error_kind == "invalid_transport_receipt" + + adapter._bot.send_document.assert_not_awaited() + fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_telegram_receipt_metadata_rejects_subclasses_before_provider_send(): + adapter = _make_adapter() + send_message = AsyncMock(return_value=SimpleNamespace(message_id=1)) + adapter._bot = SimpleNamespace(send_message=send_message) + + class HostileDict(dict): + def __bool__(self): + raise AssertionError("hostile metadata truthiness was evaluated") + + def get(self, *_args, **_kwargs): + raise AssertionError("hostile metadata get was called") + + def __contains__(self, _key): + raise AssertionError("hostile metadata membership was evaluated") + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile routing truthiness was evaluated") + + def __str__(self): + raise AssertionError("hostile routing value was stringified") + + content_result = await adapter.send( + chat_id="-100123", content=HostileText("bounded"), + metadata={ + "_transport_receipt_requested_target": { + "platform": "telegram", "chat_id": "-100123", + }, + }, + ) + outer_result = await adapter.send( + chat_id="-100123", content="bounded", + metadata=HostileDict({"_transport_receipt_requested_target": {}}), + ) + nested_result = await adapter.send( + chat_id="-100123", content="bounded", + metadata={"_transport_receipt_requested_target": HostileDict({})}, + ) + routing_result = await adapter.send( + chat_id="-100123", content="bounded", + metadata={ + "_transport_receipt_requested_target": { + "platform": "telegram", "chat_id": "-100123", + }, + "thread_id": HostileText("7"), + }, + ) + + for result in (content_result, outer_result, nested_result, routing_result): + assert result.success is False + assert result.error_kind == "invalid_transport_receipt" + send_message.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("method_name", "suffix", "bot_method", "fallback_method"), + ( + ("send_image_file", ".jpg", "send_photo", "send_document"), + ("send_document", ".pdf", "send_document", "send_document"), + ("send_video", ".mp4", "send_video", "send_video"), + ("send_voice", ".ogg", "send_voice", "send_voice"), + ("send_voice", ".mp3", "send_audio", "send_voice"), + ), +) +async def test_telegram_preregistered_media_timeout_never_falls_back( + tmp_path, monkeypatch, method_name, suffix, bot_method, fallback_method, +): + adapter = _make_adapter() + provider_send = AsyncMock(side_effect=TimeoutError("ambiguous provider outcome")) + adapter._bot = SimpleNamespace(**{bot_method: provider_send}) + fallback = AsyncMock(return_value=SendResult(success=True, message_id="fallback")) + if method_name == "send_image_file": + monkeypatch.setattr(adapter, fallback_method, fallback) + else: + monkeypatch.setattr(BasePlatformAdapter, fallback_method, fallback) + media = tmp_path / f"media{suffix}" + media.write_bytes(b"bounded-media") + + result = await getattr(adapter, method_name)( + chat_id="123", + **{ + { + "send_image_file": "image_path", + "send_document": "file_path", + "send_video": "video_path", + "send_voice": "audio_path", + }[method_name]: str(media), + }, + metadata={ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 4, + "_transport_receipt_requested_target": { + "platform": "telegram", + "chat_id": "123", + }, + }, + ) + + provider_send.assert_awaited_once() + fallback.assert_not_awaited() + assert result.success is False + assert result.retryable is False + assert result.error_kind == "unknown" + assert result.receipt is not None + assert result.receipt.outcome == "unknown" + assert (result.receipt.component, result.receipt.ordinal) == ("media", 4) + assert result.receipt.requested_target == TransportTarget("telegram", "123") + assert result.receipt.provider_message_id is None + assert result.receipt.actual_target is None + + +@pytest.mark.asyncio +async def test_telegram_photo_timeout_with_dimension_marker_never_falls_back( + tmp_path, monkeypatch, +): + adapter = _make_adapter() + provider_send = AsyncMock( + side_effect=TimeoutError( + "ambiguous provider outcome: PHOTO_INVALID_DIMENSIONS marker" + ) + ) + adapter._bot = SimpleNamespace(send_photo=provider_send) + fallback = AsyncMock(return_value=SendResult(success=True, message_id="fallback")) + monkeypatch.setattr(adapter, "send_document", fallback) + media = tmp_path / "media.jpg" + media.write_bytes(b"bounded-media") + + result = await adapter.send_image_file( + chat_id="123", + image_path=str(media), + metadata={ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 4, + "_transport_receipt_requested_target": { + "platform": "telegram", + "chat_id": "123", + }, + }, + ) + + provider_send.assert_awaited_once() + fallback.assert_not_awaited() + assert result.success is False + assert result.retryable is False + assert result.error_kind == "unknown" + assert result.receipt is not None + assert result.receipt.outcome == "unknown" + + +@pytest.mark.asyncio +async def test_telegram_exact_photo_dimensions_rejection_falls_back_once( + tmp_path, monkeypatch, +): + from telegram.error import BadRequest + + adapter = _make_adapter() + provider_send = AsyncMock(side_effect=BadRequest("PHOTO_INVALID_DIMENSIONS")) + adapter._bot = SimpleNamespace(send_photo=provider_send) + fallback_result = SendResult(success=True, message_id="fallback") + fallback = AsyncMock(return_value=fallback_result) + monkeypatch.setattr(adapter, "send_document", fallback) + media = tmp_path / "media.jpg" + media.write_bytes(b"bounded-media") + + result = await adapter.send_image_file( + chat_id="123", + image_path=str(media), + metadata={ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 4, + }, + ) + + provider_send.assert_awaited_once() + fallback.assert_awaited_once() + assert result is fallback_result + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("method_name", "suffix", "bot_method"), + ( + ("send_image_file", ".jpg", "send_photo"), + ("send_video", ".mp4", "send_video"), + ("send_voice", ".ogg", "send_voice"), + ), +) +async def test_telegram_other_native_media_acks_emit_typed_receipts( + tmp_path, method_name, suffix, bot_method, +): + adapter = _make_adapter() + provider_send = AsyncMock(return_value=SimpleNamespace(message_id=84)) + adapter._bot = SimpleNamespace(**{bot_method: provider_send}) + media = tmp_path / f"media{suffix}" + media.write_bytes(b"bounded-media") + + result = await getattr(adapter, method_name)( + chat_id="-100123", + **({ + "image_path": str(media), + } if method_name == "send_image_file" else { + "video_path": str(media), + } if method_name == "send_video" else { + "audio_path": str(media), + }), + metadata={ + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 3, + }, + ) + + assert result.success is True + assert result.receipt is not None + assert result.receipt.provider_message_id == "84" + assert (result.receipt.component, result.receipt.ordinal) == ("media", 3) + assert result.receipt.requested_target == result.receipt.actual_target + + @pytest.mark.asyncio async def test_private_dm_topic_reply_fallback_without_anchor_fails_loud(): """Anchor-required DM topic fallback must not silently send elsewhere.""" @@ -427,6 +864,8 @@ async def mock_send_media(**kwargs): "thread_id": "20197", "telegram_dm_topic_reply_fallback": True, "telegram_reply_to_message_id": "462", + "_transport_receipt_component": "media", + "_transport_receipt_ordinal": 2, }, ) @@ -436,6 +875,12 @@ async def mock_send_media(**kwargs): assert call_log[1]["reply_to_message_id"] is None assert "message_thread_id" not in call_log[1] assert "direct_messages_topic_id" not in call_log[1] + assert result.receipt is not None + assert result.receipt.provider_message_id == "782" + assert (result.receipt.component, result.receipt.ordinal) == ("media", 2) + assert result.receipt.requested_target.thread_id == "20197" + assert result.receipt.actual_target is not None + assert result.receipt.actual_target.thread_id is None @pytest.mark.asyncio diff --git a/tests/hermes_cli/test_cron.py b/tests/hermes_cli/test_cron.py index 1712e0ddab6a..cef0c1324968 100644 --- a/tests/hermes_cli/test_cron.py +++ b/tests/hermes_cli/test_cron.py @@ -12,6 +12,14 @@ from hermes_cli.subcommands.cron import build_cron_parser +def test_cli_public_timestamp_rejects_string_subclass_without_length(): + class HostileText(str): + def __len__(self): + raise AssertionError("hostile timestamp length was evaluated") + + assert cron_cli._public_timestamp(HostileText("2026-08-23T20:00:00+00:00")) is None + + @pytest.fixture() def tmp_cron_dir(tmp_path, monkeypatch): monkeypatch.setattr("cron.jobs.CRON_DIR", tmp_path / "cron") @@ -149,7 +157,7 @@ def test_list_shows_unverified_delivery(self, tmp_cron_dir, capsys): out = capsys.readouterr().out assert job["id"] in out assert "Delivery UNVERIFIED" in out - assert "slack:C0123456" in out + assert "slack:C0123456" not in out assert "without message_id/raw_response" in out def test_list_is_quiet_when_delivery_was_verified(self, tmp_cron_dir, capsys): @@ -273,9 +281,7 @@ def test_delivery_failed_is_not_green_ok(self, tmp_cron_dir, capsys, monkeypatch out = capsys.readouterr().out last_run_line = next(l for l in out.splitlines() if "Last run:" in l) assert "delivery_failed" in last_run_line - assert "telegram timeout" in last_run_line, ( - "the delivery detail lives in last_delivery_error, not last_error" - ) + assert "telegram timeout" not in out assert cron_cli.Colors.GREEN not in last_run_line def test_ok_run_still_green(self, tmp_cron_dir, capsys, monkeypatch): @@ -395,6 +401,114 @@ def test_cron_list_warns_when_gateway_not_running(monkeypatch, capsys): assert "Nightly docs" in out +def test_cron_list_redacts_legacy_error_details(monkeypatch, capsys): + monkeypatch.setattr( + "cron.jobs.list_jobs", + lambda include_disabled=False: [{ + "id": "redacted-job", + "name": "Redacted job", + "schedule_display": "every day", + "state": "scheduled", + "enabled": True, + "next_run_at": "2026-08-23T00:00:00Z", + "deliver": ["local"], + "last_status": "error", + "last_run_at": "2026-08-22T20:00:00Z", + "last_error": "RAW_LAST_ERROR_SENTINEL user@example.org", + "last_delivery_error": "RAW_DELIVERY_SENTINEL /private/report.pdf", + "last_fire_error": { + "at": "2026-08-22T19:00:00Z", + "detail": "RAW_FIRE_SENTINEL provider payload", + }, + }], + ) + monkeypatch.setattr("hermes_cli.gateway.find_gateway_pids", lambda: [1]) + monkeypatch.setattr(cron_cli, "_active_cron_provider_name", lambda: "builtin") + + cron_cli.cron_list() + + out = capsys.readouterr().out + assert "Last run:" in out and "error" in out + assert "Delivery failed" in out + assert "Missed scheduled fire" in out + assert "RAW_LAST_ERROR_SENTINEL" not in out + assert "RAW_DELIVERY_SENTINEL" not in out + assert "RAW_FIRE_SENTINEL" not in out + assert "user@example.org" not in out + assert "/private/report.pdf" not in out + + +def test_cron_list_projects_private_job_config_to_bounded_summary(monkeypatch, capsys): + sentinels = { + "deliver": "telegram:RAW_TARGET_SENTINEL", + "skills": ["PRIVATE_SKILL_SENTINEL"], + "prompt": "PRIVATE_PROMPT_SENTINEL", + "script": "PRIVATE_SCRIPT_SENTINEL", + "monitor_url": "https://PRIVATE_MONITOR_SENTINEL.invalid", + "workdir": "/PRIVATE_WORKDIR_SENTINEL", + "model": "PRIVATE_MODEL_SENTINEL", + "provider": "PRIVATE_PROVIDER_SENTINEL", + "base_url": "https://PRIVATE_BASE_URL_SENTINEL.invalid", + "profile": "PRIVATE_PROFILE_SENTINEL", + } + monkeypatch.setattr( + "cron.jobs.list_jobs", + lambda include_disabled=False: [{ + "id": "bounded-job", + "name": "Bounded job", + "schedule_display": "every day", + "state": "scheduled", + "enabled": True, + "next_run_at": "2026-08-23T00:00:00Z", + "latest_execution": { + "id": "PRIVATE_EXECUTION_ID_SENTINEL", + "status": "delivered", + }, + **sentinels, + }], + ) + monkeypatch.setattr("cron.executions.latest_execution", lambda job_id: None) + monkeypatch.setattr("hermes_cli.gateway.find_gateway_pids", lambda: [1]) + monkeypatch.setattr(cron_cli, "_active_cron_provider_name", lambda: "builtin") + + cron_cli.cron_list() + + out = capsys.readouterr().out + assert "Bounded job" in out + assert "Delivery: external" in out + assert "Mode: monitor" in out + for value in (*sentinels.values(), "PRIVATE_EXECUTION_ID_SENTINEL"): + if isinstance(value, list): + value = value[0] + assert value not in out + + +def test_cron_list_drops_malformed_fire_timestamp(monkeypatch, capsys): + sentinel = "RAW_CLI_FIRE_AT_SENTINEL user@example.org /private/report.pdf" + monkeypatch.setattr( + "cron.jobs.list_jobs", + lambda include_disabled=False: [{ + "id": "malformed-fire-at", + "name": "Malformed fire", + "schedule_display": "every day", + "state": "scheduled", + "enabled": True, + "deliver": ["local"], + "last_fire_error": {"at": sentinel, "detail": "raw detail"}, + }], + ) + monkeypatch.setattr("hermes_cli.gateway.find_gateway_pids", lambda: [1]) + monkeypatch.setattr(cron_cli, "_active_cron_provider_name", lambda: "builtin") + + cron_cli.cron_list() + + out = capsys.readouterr().out + assert "Missed scheduled fire" in out + assert sentinel not in out + assert "user@example.org" not in out + assert "/private/report.pdf" not in out + + def test_cron_tick_invokes_scheduler_tick_with_verbose(monkeypatch): calls = [] monkeypatch.setattr("cron.scheduler.tick", lambda verbose=False: calls.append(verbose)) @@ -404,6 +518,25 @@ def test_cron_tick_invokes_scheduler_tick_with_verbose(monkeypatch): assert calls == [True] +def test_cron_tick_redacts_oserror_detail(monkeypatch, capsys): + sentinel = "RAW_TICK_OSERROR_SENTINEL /private/jobs.json user@example.org" + + def fail_tick(verbose=False): + assert verbose is True + raise OSError(sentinel) + + monkeypatch.setattr("cron.scheduler.tick", fail_tick) + + result = cron_cli.cron_tick() + + out = capsys.readouterr().out + assert result == 1 + assert "Cron tick failed: tick_failed" in out + assert sentinel not in out + assert "/private/jobs.json" not in out + assert "user@example.org" not in out + + def test_cron_create_failure_returns_nonzero(monkeypatch, capsys): monkeypatch.setattr(cron_cli, "_cron_api", lambda **kwargs: {"success": False, "error": "boom"}) @@ -465,7 +598,8 @@ def test_background_dispatch_with_delegation_id_does_not_report_failed( rc, out = self._run_cmd(capsys) assert rc == 0 - assert "Running in background (delegation del-abc123)." in out + assert "Running in background." in out + assert "del-abc123" not in out assert "failed" not in out.lower() assert "Ran now" not in out @@ -546,7 +680,8 @@ def test_delegation_id_alone_counts_as_background(self, monkeypatch, capsys): rc, out = self._run_cmd(capsys) assert rc == 0 - assert "Running in background (delegation del-xyz)." in out + assert "Running in background." in out + assert "del-xyz" not in out assert "failed" not in out.lower() @@ -575,7 +710,8 @@ def test_delivery_failed_names_the_delivery_error(self, tmp_cron_dir, capsys): save_jobs(jobs) out = self._run_list(tmp_cron_dir, capsys) - assert "Last run: 2026-09-01T07:00:00+00:00 (delivery_failed: telegram: 502 Bad Gateway)" in out + assert "Last run: 2026-09-01T07:00:00+00:00 (delivery_failed)" in out + assert "telegram: 502 Bad Gateway" not in out def test_ok_stays_plain(self, tmp_cron_dir, capsys): create_job(prompt="Nightly brief", schedule="every 1h") diff --git a/tests/hermes_cli/test_cron_dashboard_off_loop.py b/tests/hermes_cli/test_cron_dashboard_off_loop.py index 636aaaff5db8..7dc00e27d5d1 100644 --- a/tests/hermes_cli/test_cron_dashboard_off_loop.py +++ b/tests/hermes_cli/test_cron_dashboard_off_loop.py @@ -15,6 +15,25 @@ from hermes_cli import web_server +def test_blueprint_list_redacts_unexpected_runtime_error(monkeypatch): + from cron import blueprint_catalog + + sentinel = "RAW_BLUEPRINT_LIST user@example.org /private/catalog" + + def explode(_blueprint): + raise OSError(sentinel) + + monkeypatch.setattr(blueprint_catalog, "blueprint_catalog_entry", explode) + monkeypatch.setattr(web_server, "_has_valid_session_token", lambda req: True) + + with TestClient(web_server.app) as client: + response = client.get("/api/cron/blueprints") + + assert response.status_code == 500 + assert response.json() == {"detail": "cron_blueprint_list_failed"} + assert sentinel not in response.text + + @pytest.fixture() def loop_probe(): """Collect (tag, on_loop) proof from stubbed profile-I/O helpers.""" @@ -57,10 +76,23 @@ def fake_find(job_id): def test_blueprint_instantiate_create_job_off_loop(monkeypatch, loop_probe): seen, probe = loop_probe + captured = {} def fake_call(profile, fn, *args, **kwargs): probe("call") - return {"id": "bp-job-1", "kwargs_seen": sorted(kwargs.keys())} + captured["kwargs_seen"] = sorted(kwargs.keys()) + return { + "id": "bp-job-1", + "name": "t", + "prompt": "hi", + "schedule": {"kind": "cron", "expr": "0 9 * * *"}, + "fire_claim": { + "by": "RAW_BLUEPRINT_OWNER user@example.org /private/owner", + }, + "hermes_home": "/private/hermes/home", + "last_output": "RAW_BLUEPRINT_OUTPUT private body", + "future_runtime_field": "RAW_BLUEPRINT_FUTURE", + } monkeypatch.setattr(web_server, "_call_cron_for_profile", fake_call) monkeypatch.setattr(web_server, "_has_valid_session_token", lambda req: True) @@ -81,7 +113,15 @@ def fake_call(profile, fn, *args, **kwargs): assert resp.status_code == 200 body = resp.json() # **spec kwargs must arrive at create_job intact through the partial. - assert body["kwargs_seen"] == ["name", "prompt", "schedule"] + assert captured["kwargs_seen"] == ["name", "prompt", "schedule"] + assert body["id"] == "bp-job-1" + assert body["name"] == "t" + assert "prompt" not in body + for private_field in ( + "fire_claim", "hermes_home", "last_output", "future_runtime_field", + ): + assert private_field not in body + assert "RAW_BLUEPRINT" not in resp.text assert ("call", False) in seen, ( f"_call_cron_for_profile must run off the event loop; proof: {seen}" ) @@ -127,3 +167,37 @@ def fail_call(profile, fn, *args, **kwargs): assert detail["scheduler_registered"] is False assert detail["retry_create"] is False assert "private callback URL and token" not in detail["error"] + + +def test_blueprint_instantiate_redacts_unexpected_runtime_error(monkeypatch): + sentinel = "RAW_BLUEPRINT_RUNTIME user@example.org /private/reconcile.json" + + monkeypatch.setattr( + web_server, + "_call_cron_for_profile", + lambda *args, **kwargs: {"id": "bp-created-job", "name": "bp job"}, + ) + + def fail_reconcile(*args, **kwargs): + raise OSError(sentinel) + + monkeypatch.setattr(web_server, "_notify_cron_provider_for_profile", fail_reconcile) + monkeypatch.setattr(web_server, "_has_valid_session_token", lambda req: True) + + import cron.blueprint_catalog as bc + monkeypatch.setattr(bc, "get_blueprint", lambda key: object()) + monkeypatch.setattr( + bc, + "fill_blueprint", + lambda bp, vals: {"name": "t", "schedule": "0 9 * * *", "prompt": "hi"}, + ) + + client = TestClient(web_server.app) + resp = client.post( + "/api/cron/blueprints/instantiate", + json={"blueprint": "morning-brief", "values": {}}, + ) + + assert resp.status_code == 400 + assert resp.json()["detail"] == "cron_create_failed" + assert sentinel not in resp.text diff --git a/tests/hermes_cli/test_cron_fire_dashboard.py b/tests/hermes_cli/test_cron_fire_dashboard.py index d6d406398d3c..36bbdeb4f96d 100644 --- a/tests/hermes_cli/test_cron_fire_dashboard.py +++ b/tests/hermes_cli/test_cron_fire_dashboard.py @@ -203,6 +203,78 @@ async def fake_forward(profile, job_id, authorization): client.close() +@pytest.mark.parametrize( + ("gateway_status", "gateway_body", "expected_body"), + [ + ( + 202, + { + "status": "accepted", + "job_id": "j-redact", + "raw": "RAW_SUCCESS_PROVIDER_SENTINEL user@example.org", + }, + {"status": "accepted", "job_id": "j-redact"}, + ), + ( + 202, + { + "status": ["RAW_UNHASHABLE_STATUS_SENTINEL"], + "raw": "RAW_UNHASHABLE_PROVIDER_SENTINEL user@example.org", + }, + { + "error": "gateway_fire_failed", + "error_kind": "gateway_fire_failed", + "job_id": "j-redact", + }, + ), + ( + 503, + { + "error": "gateway unavailable", + "reason": "RAW_FAILURE_REASON_SENTINEL /private/report.pdf", + "raw": "RAW_FAILURE_PROVIDER_SENTINEL user@example.org", + }, + { + "error": "gateway_unavailable", + "error_kind": "gateway_unavailable", + "job_id": "j-redact", + }, + ), + ], +) +def test_gateway_forward_response_is_bounded( + monkeypatch, gateway_status, gateway_body, expected_body, +): + async def fake_forward(profile, job_id, authorization): + return gateway_status, gateway_body + + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: (lambda **kw: {"purpose": "cron_fire"}), + ) + monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default") + monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward) + + client, pa, ph = _client(auth_required=False) + try: + response = client.post( + "/api/cron/fire", + headers={"Authorization": "Bearer nas-jwt"}, + json={"job_id": "j-redact"}, + ) + assert response.status_code == gateway_status + assert response.json() == expected_body + serialized = response.text + assert "RAW_" not in serialized + assert "user@example.org" not in serialized + assert "/private/report.pdf" not in serialized + if gateway_status == 503: + assert response.headers["Retry-After"] + finally: + _restore(pa, ph) + client.close() + + # ── _gateway_fire_endpoint URL resolution ──────────────────────────────── @@ -300,7 +372,7 @@ async def fake_forward(profile, job_id, authorization): "plugins.cron_providers.chronos.verify.get_fire_verifier", lambda: (lambda **kw: {"purpose": "cron_fire"}), ) - monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default") + monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "private-profile") monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward) monkeypatch.setattr( web_server, "_gateway_intentionally_stopped", lambda p: False @@ -313,6 +385,12 @@ async def fake_forward(profile, job_id, authorization): json={"job_id": "j4"}) assert resp.status_code == 503 assert resp.headers.get("Retry-After") == "60" + assert resp.json() == { + "error": "gateway_unavailable", + "error_kind": "gateway_unavailable", + "job_id": "j4", + } + assert "private-profile" not in resp.text finally: _restore(pa, ph) client.close() @@ -333,7 +411,7 @@ async def fake_forward(profile, job_id, authorization): "plugins.cron_providers.chronos.verify.get_fire_verifier", lambda: (lambda **kw: {"purpose": "cron_fire"}), ) - monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default") + monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "private-profile") monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward) monkeypatch.setattr( web_server, "_gateway_intentionally_stopped", lambda p: True @@ -347,7 +425,9 @@ async def fake_forward(profile, job_id, authorization): headers={"Authorization": "Bearer nas-jwt"}, json={"job_id": "j5"}) assert resp.status_code == 200 - assert resp.json().get("status") == "gateway_stopped" + assert resp.json() == {"status": "gateway_stopped", "job_id": "j5"} + assert "private-profile" not in resp.text + assert "detail" not in resp.json() assert executed == [] # dropped, never locally executed finally: _restore(pa, ph) diff --git a/tests/hermes_cli/test_gateway_restart_loop.py b/tests/hermes_cli/test_gateway_restart_loop.py index 6df661bc54ce..3ef9dcf16aa6 100644 --- a/tests/hermes_cli/test_gateway_restart_loop.py +++ b/tests/hermes_cli/test_gateway_restart_loop.py @@ -376,8 +376,8 @@ def test_block_hermes_gateway_restart(self, capsys): rc = cron_command(args) assert rc == 1 out = capsys.readouterr().out - assert "Blocked" in out - assert "#30719" in out + assert "cron_operation_failed" in out + assert "#30719" not in out def test_block_script_with_lifecycle_command(self, tmp_path, capsys, monkeypatch): @@ -405,7 +405,7 @@ def test_block_script_with_lifecycle_command(self, tmp_path, capsys, monkeypatch rc = cron_command(args) assert rc == 1 out = capsys.readouterr().out - assert "Blocked" in out + assert "cron_operation_failed" in out def test_allow_empty_prompt(self, capsys): @@ -1748,8 +1748,7 @@ def test_create_job_allows_benign_prompt(self): assert job["id"] def test_cronjob_tool_surfaces_block_as_error(self, tmp_path, monkeypatch): - """End-to-end through the model tool: the block comes back as - result['error'] with the #30719 hint, not an unhandled exception.""" + """The model tool exposes only the bounded failure category.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) (tmp_path / ".hermes").mkdir(parents=True) from tools.cronjob_tools import cronjob @@ -1758,7 +1757,7 @@ def test_cronjob_tool_surfaces_block_as_error(self, tmp_path, monkeypatch): prompt="please run hermes gateway restart nightly", )) assert result.get("success") is False - assert "#30719" in result.get("error", "") + assert result.get("error") == "cron_operation_failed" # --------------------------------------------------------------------------- @@ -1928,7 +1927,7 @@ def test_cron_nested_wrapper_script_is_scanned(self, tmp_path, capsys, monkeypat rc = cron_command(args) assert rc == 1 out = capsys.readouterr().out - assert "Blocked" in out + assert "cron_operation_failed" in out class TestLifecycleGuardDataArgumentExemption: """Lifecycle words inside DATA arguments (SQL text, grep patterns) must diff --git a/tests/hermes_cli/test_web_server_cron_profiles.py b/tests/hermes_cli/test_web_server_cron_profiles.py index 3a8bda9a7605..75adc4028934 100644 --- a/tests/hermes_cli/test_web_server_cron_profiles.py +++ b/tests/hermes_cli/test_web_server_cron_profiles.py @@ -7,6 +7,7 @@ import pytest from fastapi import HTTPException +from starlette.testclient import TestClient @pytest.fixture() @@ -158,13 +159,42 @@ def fail_create(*args, **kwargs): assert exc_info.value.status_code == 424 assert exc_info.value.detail == { - "error": str(failure), + "error": "scheduler_registration_failed", "job_id": "saved-job", "job_saved": True, "scheduler_registered": False, "retry_create": False, } - assert "private callback URL and token" not in str(exc_info.value.detail) + serialized = str(exc_info.value.detail) + assert "private callback URL and token" not in serialized + assert "RuntimeError" not in serialized + + +def test_dashboard_create_redacts_unexpected_runtime_error( + isolated_profiles, + monkeypatch, +): + from hermes_cli import web_server + + sentinel = "RAW_CREATE_RUNTIME_SENTINEL user@example.org /private/store.json" + + def fail_create(*args, **kwargs): + raise OSError(sentinel) + + monkeypatch.setattr(web_server, "_call_cron_for_profile", fail_create) + + with pytest.raises(HTTPException) as exc_info: + web_server._create_cron_job_sync( + web_server.CronJobCreate( + prompt="managed by named profile", + schedule="every 1h", + ), + profile="worker_alpha", + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == "cron_create_failed" + assert sentinel not in str(exc_info.value.detail) def test_notify_cron_provider_scopes_store_and_runtime_home_together( @@ -425,29 +455,649 @@ def run_ticker_write(): -@pytest.mark.asyncio -async def test_cron_mutation_without_profile_finds_named_profile_job(isolated_profiles): +def test_cron_mutations_require_concrete_profile(monkeypatch): + from fastapi.testclient import TestClient from hermes_cli import web_server - worker_job = web_server._call_cron_for_profile( - "worker_alpha", - "create_job", - prompt="managed by named profile", - schedule="every 1h", - name="named-profile-job", + calls = [] + + def forbidden(*args, **kwargs): + calls.append((args, kwargs)) + raise AssertionError("mutation boundary reached without a concrete profile") + + monkeypatch.setattr(web_server, "_has_valid_session_token", lambda _request: True) + monkeypatch.setattr(web_server, "_find_cron_job_profile", forbidden) + monkeypatch.setattr(web_server, "_mutate_cron_for_profile", forbidden) + monkeypatch.setattr(web_server, "_call_cron_for_profile", forbidden) + monkeypatch.setattr(web_server, "_fire_cron_job_for_profile", forbidden) + + requests = ( + ("post", "/api/cron/jobs", {"json": {"prompt": "x", "schedule": "every 1h"}}), + ("put", "/api/cron/jobs/job-1", {"json": {"updates": {"name": "x"}}}), + ("post", "/api/cron/jobs/job-1/pause", {}), + ("post", "/api/cron/jobs/job-1/resume", {}), + ("post", "/api/cron/jobs/job-1/trigger", {}), + ("delete", "/api/cron/jobs/job-1", {}), ) - paused = await web_server.pause_cron_job(worker_job["id"]) - assert paused["profile"] == "worker_alpha" - assert paused["enabled"] is False + with TestClient(web_server.app) as client: + for method, path, kwargs in requests: + missing = getattr(client, method)(path, **kwargs) + aggregate = getattr(client, method)(f"{path}?profile=all", **kwargs) + assert missing.status_code == 422 + assert aggregate.status_code == 400 - default_jobs = await web_server.list_cron_jobs(profile="default") - worker_jobs = await web_server.list_cron_jobs(profile="worker_alpha") + assert calls == [] + + +def test_public_cron_profile_metadata_is_canonical_and_route_derived(): + from hermes_cli import web_server + + public = web_server._public_cron_job_for_profile( + { + "id": "same-id", + "profile": "poisoned-profile", + "profile_name": "Poisoned Profile", + "hermes_home": "/private/home", + }, + "Worker_Alpha", + ) + + assert public["profile"] == "worker_alpha" + assert public["profile_name"] == "worker_alpha" + assert public["is_default_profile"] is False + assert "hermes_home" not in public + assert "poisoned" not in json.dumps(public, sort_keys=True).lower() + + +def test_dashboard_cron_summary_and_detail_have_separate_trust_boundaries(monkeypatch): + from hermes_cli import web_server + + raw_job = { + "id": "redacted-job", + "name": "bounded summary name", + "prompt": "PRIVATE_PROMPT_SENTINEL user@example.org\nsecond line", + "script": "private/script.py", + "workdir": "/private/worktree", + "model": "private-model", + "provider": "private-provider", + "provider_snapshot": "private-provider-snapshot", + "model_snapshot": "private-model-snapshot", + "base_url": "https://private-provider.example.org/v1", + "profile": "worker-private", + "profile_name": "Worker Private", + "skills": ["private-skill"], + "context_from": ["private-upstream-job"], + "enabled_toolsets": ["private-toolset"], + "deliver": "matrix:private-room-id", + "no_agent": True, + "monitor_url": "https://private-monitor.example.org/feed", + "last_status": "error", + "last_error": "RAW_LAST_ERROR_SENTINEL user@example.org", + "last_delivery_error": "RAW_DELIVERY_SENTINEL /private/report.pdf", + "last_fire_error": { + "at": "2026-08-22T19:00:00Z", + "detail": "RAW_FIRE_SENTINEL provider payload", + }, + "fire_claim": { + "by": "RAW_CLAIM_OWNER_SENTINEL user@example.org /private/owner", + "at": "2026-08-22T19:01:00Z", + "fire_at": "2026-08-22T19:00:00Z", + }, + "execution_id": "RAW_EXECUTION_SENTINEL", + "fire_identity": "RAW_FIRE_IDENTITY_SENTINEL", + "last_output": "RAW_OUTPUT_SENTINEL private result body", + "hermes_home": "/private/hermes/home", + "future_runtime_field": "RAW_FUTURE_RUNTIME_SENTINEL", + } + + def call(_profile, func_name, *_args): + if func_name == "list_jobs": + return [raw_job] + if func_name == "get_job": + return raw_job + raise AssertionError(func_name) + + monkeypatch.setattr(web_server, "_call_cron_for_profile", call) + + listed = web_server._list_cron_jobs_sync("default")[0] + fetched = web_server._get_cron_job_sync("redacted-job", profile="default") + sensitive_summary_fields = { + "prompt", "script", "workdir", "model", "provider", "base_url", + "provider_snapshot", "model_snapshot", + "skills", "context_from", "enabled_toolsets", "deliver", "no_agent", + "monitor_url", + } + for public in (listed, fetched): + serialized = json.dumps(public, sort_keys=True) + assert public["name"] == "bounded summary name" + assert public["profile"] == "default" + assert public["profile_name"] == "default" + assert public["is_default_profile"] is True + assert public["delivery_kind"] == "external" + assert public["mode"] == "monitor" + assert public["skill_count"] == 1 + assert public["toolset_count"] == 1 + assert public["model_configured"] is True + assert sensitive_summary_fields.isdisjoint(public) + assert public["last_error"] == "run_failed" + assert public["last_delivery_error"] == "delivery_failed" + assert public["last_fire_error"] == { + "at": "2026-08-22T19:00:00Z", + "error_kind": "fire_forward_failed", + } + assert "RAW_" not in serialized + assert "PRIVATE_" not in serialized + assert "worker-private" not in serialized + assert "Worker Private" not in serialized + assert "user@example.org" not in serialized + assert "/private/" not in serialized + + detail = web_server._get_cron_job_detail_sync( + "redacted-job", profile="default", + ) + assert detail["prompt"] == raw_job["prompt"] + assert detail["script"] == raw_job["script"] + assert detail["workdir"] == raw_job["workdir"] + assert detail["model"] == raw_job["model"] + assert detail["provider"] == raw_job["provider"] + assert detail["base_url"] == raw_job["base_url"] + assert detail["deliver"] == raw_job["deliver"] + assert detail["no_agent"] is True + assert detail["monitor_url"] == raw_job["monitor_url"] + assert "profile" not in detail + assert "profile_name" not in detail + assert "provider_snapshot" not in detail + assert "model_snapshot" not in detail + assert "fire_claim" not in detail + assert "last_output" not in detail + assert "future_runtime_field" not in detail + + assert raw_job["last_error"].startswith("RAW_LAST_ERROR_SENTINEL") + assert "detail" in raw_job["last_fire_error"] + + +def test_dashboard_cron_detail_endpoint_requires_explicit_profile(monkeypatch): + from hermes_cli import web_server + + raw_job = { + "id": "detail-job", + "name": "detail job", + "prompt": "PRIVATE_DETAIL_PROMPT", + "script": "private/detail.py", + "workdir": "/private/detail-worktree", + "enabled": True, + "state": "scheduled", + } + + calls = [] + + def call(profile, func_name, *args): + calls.append((profile, func_name, args)) + return raw_job + + monkeypatch.setattr(web_server, "_has_valid_session_token", lambda _request: True) + monkeypatch.setattr(web_server, "_call_cron_for_profile", call) + + with TestClient(web_server.app) as client: + missing = client.get("/api/cron/jobs/detail-job/detail") + aggregate = client.get("/api/cron/jobs/detail-job/detail?profile=all") + detail = client.get( + "/api/cron/jobs/detail-job/detail?profile=worker_alpha", + ) + + assert missing.status_code == 422 + assert aggregate.status_code == 400 + assert detail.status_code == 200 + assert calls == [("worker_alpha", "get_job", ("detail-job",))] + assert detail.json()["prompt"] == "PRIVATE_DETAIL_PROMPT" + assert detail.json()["workdir"] == "/private/detail-worktree" + assert "profile" not in detail.json() + + +def test_dashboard_cron_job_surface_drops_malformed_fire_error(monkeypatch): + from hermes_cli import web_server + + sentinel = "RAW_MALFORMED_FIRE_SENTINEL user@example.org /private/report.pdf" + raw_job = {"id": "malformed-fire", "last_fire_error": sentinel} + monkeypatch.setattr( + web_server, + "_call_cron_for_profile", + lambda _profile, _func_name, *_args: [raw_job], + ) + + public = web_server._list_cron_jobs_sync("default")[0] + assert public["last_fire_error"] is None + assert sentinel not in json.dumps(public, sort_keys=True) + assert raw_job["last_fire_error"] == sentinel + + +@pytest.mark.parametrize( + "malformed_deliver", + [ + ["PRIVATE_DELIVERY_LIST_SENTINEL"], + {"private": "PRIVATE_DELIVERY_DICT_SENTINEL"}, + ], +) +def test_dashboard_cron_job_surface_rejects_unhashable_delivery(malformed_deliver): + from hermes_cli import web_server + + public = web_server._public_cron_job({ + "id": "malformed-delivery", + "deliver": malformed_deliver, + }) + + assert public["delivery_kind"] == "local" + assert "PRIVATE_DELIVERY" not in json.dumps(public, sort_keys=True) + + +def test_dashboard_cron_projection_rejects_builtin_subclasses_before_magic_methods(): + from hermes_cli import web_server + + class HostileDict(dict): + def get(self, *_args, **_kwargs): + raise AssertionError("hostile mapping get was called") + + def __contains__(self, _key): + raise AssertionError("hostile mapping membership was evaluated") + + def __getitem__(self, _key): + raise AssertionError("hostile mapping item access was called") + + class HostileList(list): + def __iter__(self): + raise AssertionError("hostile list was iterated") + + def __len__(self): + raise AssertionError("hostile list length was evaluated") + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile text truthiness was evaluated") + + def __len__(self): + raise AssertionError("hostile text length was evaluated") + + with pytest.raises(TypeError, match="requires an object"): + web_server._public_cron_job(HostileDict({"id": "hostile"})) + + public = web_server._public_cron_job({ + "id": "bounded", + "schedule": HostileDict({"kind": "once"}), + "repeat": HostileDict({"times": 1}), + "last_fire_error": HostileDict({"at": "2026-08-23T20:00:00+00:00"}), + "skills": HostileList(["private"]), + "enabled_toolsets": HostileList(["private"]), + "deliver": HostileText("external:private"), + "model": HostileText("private-model"), + }) + + assert public["schedule"] is None + assert public["repeat"] is None + assert public["last_fire_error"] is None + assert public["skill_count"] == 0 + assert public["toolset_count"] == 0 + assert public["delivery_kind"] == "local" + assert public["model_configured"] is False + + +def test_dashboard_cron_projection_rejects_bool_class_spoof_without_descriptor_call(): + from hermes_cli import web_server + + class_property_calls = [] + + class HostileBoolSpoof: + @property + def __class__(self): + class_property_calls.append("called") + return bool + + summary = web_server._public_cron_job({ + "id": "bounded", + "enabled": HostileBoolSpoof(), + }) + detail = web_server._public_cron_job_detail({ + "id": "bounded", + "enabled": HostileBoolSpoof(), + "no_agent": HostileBoolSpoof(), + "continuity": HostileBoolSpoof(), + }) + + assert summary["enabled"] is False + assert detail["enabled"] is False + assert detail["no_agent"] is False + assert detail["continuity"] is False + assert class_property_calls == [] + + +def test_cron_run_and_gateway_fire_projections_reject_dict_subclasses_without_get(): + from hermes_cli import web_server + from hermes_cli.web_routers.cron import _public_gateway_fire_body + + class HostileDict(dict): + def get(self, *_args, **_kwargs): + raise AssertionError("hostile projection get was called") + + assert web_server._public_cron_run(HostileDict({"id": "run"}), now=1.0) is None + assert _public_gateway_fire_body(200, HostileDict({"status": "accepted"}), "job") == { + "error": "gateway_fire_failed", + "error_kind": "gateway_fire_failed", + "job_id": "job", + } + + +def test_dashboard_cron_job_surface_drops_malformed_timestamps(monkeypatch): + from hermes_cli import web_server + + sentinel = "RAW_TIMESTAMP_SENTINEL user@example.org /private/runtime" + raw_job = { + "id": "malformed-timestamps", + "last_run_at": {"raw": sentinel}, + "next_run_at": sentinel, + "schedule": { + "kind": "once", + "run_at": sentinel, + "future_nested_runtime": sentinel, + }, + } + monkeypatch.setattr( + web_server, + "_call_cron_for_profile", + lambda _profile, _func_name, *_args: [raw_job], + ) + + public = web_server._list_cron_jobs_sync("default")[0] + + assert public["last_run_at"] is None + assert public["next_run_at"] is None + assert public["schedule"]["run_at"] is None + assert "future_nested_runtime" not in public["schedule"] + assert sentinel not in json.dumps(public, sort_keys=True) + + aware = "2026-08-23T08:00:00+00:00" + valid = web_server._public_cron_job({ + "id": "valid-timestamps", + "last_run_at": aware, + "next_run_at": aware, + "schedule": {"kind": "once", "run_at": aware}, + }) + assert valid["last_run_at"] == aware + assert valid["next_run_at"] == aware + assert valid["schedule"]["run_at"] == aware + + +def test_dashboard_cron_mutation_surfaces_redact_raw_error_details(monkeypatch, tmp_path): + from hermes_cli import web_server + + raw_job = { + "id": "mutation-redaction-job", + "name": "mutation redaction", + "prompt": "safe mutation prompt", + "enabled": True, + "state": "scheduled", + "last_run_at": "2026-08-22T22:00:00+00:00", + "last_status": "error", + "last_error": "RAW_MUTATION_RUNTIME_SENTINEL provider says secret", + "last_delivery_error": "RAW_MUTATION_DELIVERY_SENTINEL /private/media.pdf", + "last_fire_error": { + "at": "2026-08-22T22:01:00+00:00", + "detail": "RAW_MUTATION_FIRE_SENTINEL user@example.org", + }, + "fire_claim": { + "by": "RAW_MUTATION_CLAIM_SENTINEL user@example.org /private/owner", + "at": "2026-08-22T22:01:00+00:00", + "fire_at": "2026-08-22T22:00:00+00:00", + }, + "execution_id": "RAW_MUTATION_EXECUTION_SENTINEL", + "fire_identity": "RAW_MUTATION_IDENTITY_SENTINEL", + "last_output": "RAW_MUTATION_OUTPUT_SENTINEL private body", + "future_runtime_field": "RAW_MUTATION_FUTURE_SENTINEL", + } + + monkeypatch.setattr( + web_server, + "_cron_profile_home", + lambda profile: (profile, tmp_path), + ) + monkeypatch.setattr( + web_server, + "_call_cron_for_profile", + lambda _profile, _func_name, *_args: raw_job, + ) + monkeypatch.setattr( + web_server, + "_mutate_cron_for_profile", + lambda _profile, _func_name, *_args, **_kwargs: raw_job, + ) + monkeypatch.setattr(web_server, "_fire_cron_job_for_profile", lambda *_args, **_kwargs: True) + + responses = [ + web_server._create_cron_job_sync( + web_server.CronJobCreate( + prompt="safe mutation prompt", + schedule="every 1h", + name="mutation redaction", + ), + profile="default", + ), + web_server._update_cron_job_sync( + raw_job["id"], + web_server.CronJobUpdate(updates={"name": "updated"}), + profile="default", + ), + web_server._pause_cron_job_sync(raw_job["id"], profile="default"), + web_server._resume_cron_job_sync(raw_job["id"], profile="default"), + web_server._trigger_cron_job_sync(raw_job["id"], profile="default"), + ] + + for public in responses: + assert "prompt" not in public + assert public["last_error"] == "run_failed" + assert public["last_delivery_error"] == "delivery_failed" + assert public["last_fire_error"] == { + "at": "2026-08-22T22:01:00+00:00", + "error_kind": "fire_forward_failed", + } + serialized = json.dumps(public, sort_keys=True) + assert "RAW_MUTATION" not in serialized + assert "user@example.org" not in serialized + assert "/private/media.pdf" not in serialized + for private_field in ( + "fire_claim", "execution_id", "fire_identity", "last_output", + "future_runtime_field", + ): + assert private_field not in public + + assert raw_job["last_error"].startswith("RAW_MUTATION_RUNTIME_SENTINEL") + assert "detail" in raw_job["last_fire_error"] + + +def test_dashboard_update_delete_redact_unexpected_value_errors(monkeypatch, tmp_path): + from hermes_cli import web_server + + sentinel = "RAW_UPDATE_DELETE user@example.org /private/cron/jobs.json" + existing = { + "id": "bounded-mutation-errors", + "prompt": "safe prompt", + "schedule": {"kind": "cron", "expr": "0 9 * * *"}, + } + + monkeypatch.setattr(web_server, "_has_valid_session_token", lambda _request: True) + monkeypatch.setattr( + web_server, + "_cron_profile_home", + lambda profile: (profile, tmp_path), + ) + monkeypatch.setattr( + web_server, + "_call_cron_for_profile", + lambda _profile, function, *_args: existing + if function == "get_job" + else None, + ) + + def explode(*_args, **_kwargs): + raise ValueError(sentinel) + + monkeypatch.setattr(web_server, "_mutate_cron_for_profile", explode) + + with TestClient(web_server.app) as client: + updated = client.put( + f"/api/cron/jobs/{existing['id']}?profile=default", + json={"updates": {"name": "updated"}}, + ) + deleted = client.delete( + f"/api/cron/jobs/{existing['id']}?profile=default", + ) - assert default_jobs == [] - assert len(worker_jobs) == 1 - assert worker_jobs[0]["id"] == worker_job["id"] - assert worker_jobs[0]["enabled"] is False + assert updated.status_code == 400 + assert updated.json() == {"detail": "cron_update_failed"} + assert deleted.status_code == 400 + assert deleted.json() == {"detail": "cron_delete_failed"} + assert sentinel not in updated.text + assert sentinel not in deleted.text + + +def test_dashboard_cron_run_history_uses_bounded_projection(monkeypatch): + from hermes_cli import web_server + + sentinels = { + "system_prompt": "RAW_SYSTEM_PROMPT user@example.org", + "preview": "RAW_USER_MESSAGE /private/transcript.txt", + "cwd": "/private/worktree", + "model": "private-provider/private-model", + "profile": "private-profile", + "future_runtime": object(), + } + raw_run = { + "id": "cron_bounded-history_20260823_100000", + "source": "cron", + "started_at": 1_700_000_000.0, + "ended_at": None, + "last_active": 1_700_000_001.0, + "end_reason": None, + "archived": 0, + **sentinels, + } + malformed_run = { + "id": "cron_bounded-history_20260823_100001", + "started_at": True, + "ended_at": "RAW_END_TIME /private/time", + "last_active": float("nan"), + "end_reason": {"private": "RAW_END_REASON user@example.org"}, + "archived": "1", + } + unsafe_id_run = { + "id": "cron_user@example.org/private", + "started_at": 1_700_000_002.0, + "ended_at": None, + "last_active": 1_700_000_002.0, + } + + class FakeSessionDB: + def list_cron_job_runs(self, job_id, *, limit, offset): + assert job_id == "bounded-history" + assert (limit, offset) == (20, 0) + return [dict(raw_run), malformed_run, unsafe_id_run] + + def close(self): + pass + + monkeypatch.setattr(web_server, "_has_valid_session_token", lambda _request: True) + monkeypatch.setattr( + web_server, + "_call_cron_for_profile", + lambda _profile, function, *_args: {"id": "bounded-history"} + if function == "get_job" + else None, + ) + monkeypatch.setattr( + web_server, + "_open_session_db_for_profile", + lambda _profile, read_only: FakeSessionDB(), + ) + monkeypatch.setattr(web_server.time, "time", lambda: 1_700_000_100.0) + + with TestClient(web_server.app) as client: + response = client.get( + "/api/cron/jobs/bounded-history/runs?profile=default", + ) + + assert response.status_code == 200 + assert response.json() == { + "runs": [ + { + "id": raw_run["id"], + "status": "running", + "started_at": 1_700_000_000.0, + "ended_at": None, + "last_active": 1_700_000_001.0, + "is_active": True, + "archived": False, + }, + { + "id": malformed_run["id"], + "status": "ended", + "started_at": None, + "ended_at": None, + "last_active": None, + "is_active": False, + "archived": False, + }, + ], + "limit": 20, + } + serialized = response.text + for sentinel in sentinels.values(): + if isinstance(sentinel, str): + assert sentinel not in serialized + assert "RAW_END_TIME" not in serialized + assert "RAW_END_REASON" not in serialized + assert "user@example.org/private" not in serialized + + +def test_dashboard_trigger_completed_fallback_redacts_raw_error_details(monkeypatch): + from hermes_cli import web_server + + raw_job = { + "id": "completed-trigger-redaction-job", + "name": "completed trigger redaction", + "enabled": True, + "state": "scheduled", + "last_run_at": None, + "last_status": "error", + "last_error": "RAW_COMPLETED_RUNTIME_SENTINEL provider says secret", + "last_delivery_error": "RAW_COMPLETED_DELIVERY_SENTINEL /private/media.pdf", + "last_fire_error": { + "at": "2026-08-22T22:02:00+00:00", + "detail": "RAW_COMPLETED_FIRE_SENTINEL user@example.org", + }, + } + + def read_job(_profile, function, *_args): + if function == "resolve_job_ref": + return raw_job + if function == "get_job": + return None + raise AssertionError(function) + + monkeypatch.setattr(web_server, "_call_cron_for_profile", read_job) + monkeypatch.setattr(web_server, "_fire_cron_job_for_profile", lambda *_args, **_kwargs: True) + + public = web_server._trigger_cron_job_sync(raw_job["id"], profile="default") + + assert public["enabled"] is False + assert public["state"] == "completed" + assert public["last_error"] == "run_failed" + assert public["last_delivery_error"] == "delivery_failed" + assert public["last_fire_error"] == { + "at": "2026-08-22T22:02:00+00:00", + "error_kind": "fire_forward_failed", + } + serialized = json.dumps(public, sort_keys=True) + assert "RAW_COMPLETED" not in serialized + assert "user@example.org" not in serialized + assert "/private/media.pdf" not in serialized @pytest.mark.asyncio @@ -506,7 +1156,11 @@ async def test_blueprint_instantiation_notifies_selected_profile_provider( profile="worker_alpha", ) - assert created["profile"] == "worker_alpha" + assert "profile" not in created + persisted = web_server._call_cron_for_profile( + "worker_alpha", "get_job", created["id"], + ) + assert persisted["profile"] == "worker_alpha" assert notified_profiles == ["worker_alpha"] @@ -727,7 +1381,8 @@ def fire_due(self, job_id, *, adapters=None, loop=None, force=False): ) assert triggered["last_status"] == "error" - assert triggered["last_error"] == "expected failure" + assert triggered["last_error"] == "run_failed" + assert "expected failure" not in json.dumps(triggered, sort_keys=True) @pytest.mark.asyncio @@ -793,24 +1448,30 @@ async def test_cron_profile_scan_runs_off_event_loop(isolated_profiles, monkeypa profile_scan_threads = SimpleQueue() worker_threads = SimpleQueue() original_profile_dicts = web_server._cron_profile_dicts - original_find = web_server._find_cron_job_profile + original_mutate = web_server._mutate_cron_for_profile def tracking_profile_dicts(): profile_scan_threads.put(threading.get_ident()) return original_profile_dicts() - def tracking_find(job_id): + def tracking_mutate(profile, func_name, *args, **kwargs): worker_threads.put(threading.get_ident()) - return original_find(job_id) + return original_mutate(profile, func_name, *args, **kwargs) monkeypatch.setattr(web_server, "_cron_profile_dicts", tracking_profile_dicts) - monkeypatch.setattr(web_server, "_find_cron_job_profile", tracking_find) + monkeypatch.setattr(web_server, "_mutate_cron_for_profile", tracking_mutate) jobs = await web_server.list_cron_jobs(profile="all") - paused = await web_server.pause_cron_job(worker_job["id"]) + paused = await web_server.pause_cron_job( + worker_job["id"], profile="worker_alpha", + ) - assert any(job["id"] == worker_job["id"] for job in jobs) - assert paused["profile"] == "worker_alpha" + listed_worker = next(job for job in jobs if job["id"] == worker_job["id"]) + assert listed_worker["profile"] == "worker_alpha" + assert listed_worker["profile_name"] == "worker_alpha" + assert listed_worker["is_default_profile"] is False + assert "hermes_home" not in listed_worker + assert "profile" not in paused profile_scan_thread_ids = _drain_queue(profile_scan_threads) worker_thread_ids = _drain_queue(worker_threads) assert profile_scan_thread_ids @@ -859,10 +1520,16 @@ async def test_update_cron_job_normalizes_dashboard_core_fields(isolated_profile profile="worker_alpha", ) - assert updated["base_url"] == "https://example.invalid/v1" - assert updated["script"] == "collect.py" - assert updated["context_from"] is None - assert updated["no_agent"] is True + assert updated["mode"] == "script" + for field in ("base_url", "script", "context_from", "no_agent"): + assert field not in updated + detail = web_server._get_cron_job_detail_sync( + job["id"], profile="worker_alpha", + ) + assert detail["base_url"] == "https://example.invalid/v1" + assert detail["script"] == "collect.py" + assert detail["context_from"] == [] + assert detail["no_agent"] is True @pytest.mark.asyncio @@ -924,8 +1591,14 @@ async def test_update_cron_job_no_agent_reuses_existing_script(isolated_profiles profile="worker_alpha", ) - assert updated["no_agent"] is True - assert updated["script"] == "collect.py" + assert updated["mode"] == "script" + assert "no_agent" not in updated + assert "script" not in updated + detail = web_server._get_cron_job_detail_sync( + job["id"], profile="worker_alpha", + ) + assert detail["no_agent"] is True + assert detail["script"] == "collect.py" @pytest.mark.asyncio @@ -1018,8 +1691,13 @@ async def test_dashboard_cron_noop_inference_fields_keep_existing_snapshots( ) assert updated["name"] == "dashboard-edit-job-renamed" - assert updated["provider_snapshot"] == "initial-provider" - assert updated["model_snapshot"] == "test-model" + assert "provider_snapshot" not in updated + assert "model_snapshot" not in updated + persisted = web_server._call_cron_for_profile( + "worker_alpha", "get_job", job["id"], + ) + assert persisted["provider_snapshot"] == "initial-provider" + assert persisted["model_snapshot"] == "test-model" @pytest.mark.asyncio @@ -1060,8 +1738,13 @@ async def test_update_cron_job_clears_snapshots_for_no_agent( profile="worker_alpha", ) - assert updated["provider_snapshot"] is None - assert updated["model_snapshot"] is None + assert "provider_snapshot" not in updated + assert "model_snapshot" not in updated + persisted = web_server._call_cron_for_profile( + "worker_alpha", "get_job", job["id"], + ) + assert persisted["provider_snapshot"] is None + assert persisted["model_snapshot"] is None @pytest.mark.asyncio @@ -1140,12 +1823,10 @@ async def test_cron_profile_validation_errors(isolated_profiles): @pytest.mark.asyncio -async def test_create_cron_job_without_profile_uses_backend_own_profile( +async def test_create_cron_job_with_explicit_worker_profile_uses_worker_store( isolated_profiles, monkeypatch ): - """A pool backend scoped to a named profile must not default creates to - ``~/.hermes`` when the request carries no explicit ``profile`` (the - Desktop app's pre-profileScoped clients sent none).""" + """An explicit named profile must write only that profile's store.""" from hermes_cli import web_server monkeypatch.setenv( @@ -1158,20 +1839,23 @@ async def test_create_cron_job_without_profile_uses_backend_own_profile( schedule="every 1h", name="own-profile-job", ), - profile=None, + profile="worker_alpha", ) - assert job["profile"] == "worker_alpha" + assert "profile" not in job + persisted = web_server._call_cron_for_profile( + "worker_alpha", "get_job", job["id"], + ) + assert persisted["profile"] == "worker_alpha" assert (isolated_profiles["worker_alpha"] / "cron" / "jobs.json").exists() assert not (isolated_profiles["default"] / "cron" / "jobs.json").exists() @pytest.mark.asyncio -async def test_create_cron_job_without_profile_defaults_when_unscoped( +async def test_create_cron_job_with_explicit_default_uses_default_store( isolated_profiles, monkeypatch ): - """HERMES_HOME at the default home (or unrecognized) keeps the legacy - ``default`` fallback.""" + """An explicit default profile writes the default store.""" from hermes_cli import web_server monkeypatch.setenv("HERMES_HOME", str(isolated_profiles["default"])) @@ -1182,8 +1866,12 @@ async def test_create_cron_job_without_profile_defaults_when_unscoped( schedule="every 1h", name="default-job", ), - profile=None, + profile="default", ) - assert job["profile"] == "default" + assert "profile" not in job + persisted = web_server._call_cron_for_profile( + "default", "get_job", job["id"], + ) + assert persisted["profile"] == "default" assert (isolated_profiles["default"] / "cron" / "jobs.json").exists() diff --git a/tests/hermes_cli/test_web_server_skill_editor.py b/tests/hermes_cli/test_web_server_skill_editor.py index b5e9a7533aee..18f1b553017b 100644 --- a/tests/hermes_cli/test_web_server_skill_editor.py +++ b/tests/hermes_cli/test_web_server_skill_editor.py @@ -167,6 +167,7 @@ class TestCronJobSkills: def test_create_job_with_skills(self, client, isolated_profiles): resp = client.post( "/api/cron/jobs", + params={"profile": "default"}, json={ "prompt": "do work", "schedule": "every 1h", @@ -176,19 +177,22 @@ def test_create_job_with_skills(self, client, isolated_profiles): ) assert resp.status_code == 200 job = resp.json() - assert job["skills"] == ["dashboard-skill"] + assert "skills" not in job - # Round-trip: the list endpoint carries the skills field too. - listed = client.get("/api/cron/jobs", params={"profile": "default"}).json() - match = [j for j in listed if j["id"] == job["id"]] - assert match and match[0]["skills"] == ["dashboard-skill"] + # Sensitive editable configuration is available only from concrete-profile detail. + detail = client.get( + f"/api/cron/jobs/{job['id']}/detail", + params={"profile": "default"}, + ).json() + assert detail["skills"] == ["dashboard-skill"] def test_update_job_skills(self, client, isolated_profiles): job = client.post( "/api/cron/jobs", + params={"profile": "default"}, json={"prompt": "do work", "schedule": "every 1h"}, ).json() - assert job.get("skills") in (None, []) + assert "skills" not in job resp = client.put( f"/api/cron/jobs/{job['id']}", @@ -196,7 +200,12 @@ def test_update_job_skills(self, client, isolated_profiles): params={"profile": "default"}, ) assert resp.status_code == 200 - assert resp.json()["skills"] == ["dashboard-skill", "worker-skill"] + assert "skills" not in resp.json() + detail = client.get( + f"/api/cron/jobs/{job['id']}/detail", + params={"profile": "default"}, + ).json() + assert detail["skills"] == ["dashboard-skill", "worker-skill"] # Clearing works too. resp = client.put( @@ -205,4 +214,9 @@ def test_update_job_skills(self, client, isolated_profiles): params={"profile": "default"}, ) assert resp.status_code == 200 - assert resp.json()["skills"] == [] + assert "skills" not in resp.json() + detail = client.get( + f"/api/cron/jobs/{job['id']}/detail", + params={"profile": "default"}, + ).json() + assert detail["skills"] == [] diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py index 67200f313006..b59614a9db09 100644 --- a/tests/monitoring/test_cron_health_export.py +++ b/tests/monitoring/test_cron_health_export.py @@ -43,7 +43,15 @@ def test_execution_projection_is_opaque_bounded_and_content_free(): assert "top-secret-token" not in str(event) +def test_execution_projection_preserves_conservative_unknown_delivery_outcome(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + {"job_id": "opaque", "source": "direct", "status": "completed"}, + delivery_outcome="unknown", + ).to_dict() + assert event["delivery_outcome"] == "unknown" diff --git a/tests/plugins/test_chronos_cron.py b/tests/plugins/test_chronos_cron.py index 75971ea2d309..0e0cafffcd40 100644 --- a/tests/plugins/test_chronos_cron.py +++ b/tests/plugins/test_chronos_cron.py @@ -139,7 +139,10 @@ def test_fire_due_rearms_next_oneshot(chronos, monkeypatch): def test_fire_due_rearms_after_claimed_job_failure(chronos, monkeypatch): """A claimed attempt is consumed even when the job pipeline reports failure.""" prov, fake = chronos - claimed = {"id": "j1", "fire_claim": {"by": "owner-1"}} + claimed = { + "id": "j1", + "fire_claim": {"by": "owner-1", "fire_at": "2026-06-18T12:00:00+00:00"}, + } persisted = { "id": "j1", "enabled": True, @@ -151,6 +154,10 @@ def test_fire_due_rearms_after_claimed_job_failure(chronos, monkeypatch): "cron.executions.create_execution", lambda jid, source: {"id": "exec-1"}, ) + monkeypatch.setattr( + "cron.executions.bind_execution_fire_identity", + lambda eid, fire: {"id": eid, "fire_identity": fire}, + ) monkeypatch.setattr("cron.scheduler.run_one_job", lambda *args, **kwargs: False) monkeypatch.setattr("cron.jobs.get_job", lambda jid: persisted) diff --git a/tests/tools/test_cronjob_run_background.py b/tests/tools/test_cronjob_run_background.py index c120273c03e9..a43faefd53d7 100644 --- a/tests/tools/test_cronjob_run_background.py +++ b/tests/tools/test_cronjob_run_background.py @@ -17,6 +17,7 @@ from unittest.mock import patch from tools.cronjob_tools import ( + _format_job, _try_dispatch_background_run, cronjob, ) @@ -150,7 +151,8 @@ def test_failed_run_reports_error_status_in_event(self): time.sleep(0.05) assert found is not None assert found["status"] == "error" - assert "provider exploded" in (found.get("error") or "") + assert found.get("error") == "run_failed" + assert "provider exploded" not in json.dumps(found, sort_keys=True) def test_claim_lost_reports_immediately_without_dispatch(self): """Paused/already-firing jobs report in the tool response, not as a @@ -182,8 +184,18 @@ def test_async_delivery_unsupported_falls_back_to_sync(self): def test_pool_at_capacity_runs_inline(self): """A rejected dispatch must not strand the already-taken claim.""" + claimed = { + **_job("job-bg-07"), + "fire_claim": { + "by": "bg-owner", + "at": "2026-08-22T22:00:00+00:00", + "fire_at": "2026-08-22T21:59:00+00:00", + }, + "execution_id": "exec-bg-07", + "fire_identity": "fire-bg-07", + } with _bound_session_key(): - with patch("tools.cronjob_tools.claim_job_for_fire", side_effect=lambda jid, **kw: {**_job(jid), "fire_claim": {"by": "bg-owner"}}), \ + with patch("tools.cronjob_tools.claim_job_for_fire", return_value=claimed), \ patch("tools.async_delegation.dispatch_async_delegation", return_value={"status": "rejected", "error": "capacity"}), \ patch("cron.scheduler.run_one_job", return_value=True) as m_run, \ @@ -192,7 +204,9 @@ def test_pool_at_capacity_runs_inline(self): res = _try_dispatch_background_run(_job('job-bg-07')) assert res["dispatched"] is False assert res["success"] is True - m_run.assert_called_once() # ran inline on this thread + m_run.assert_called_once() + assert m_run.call_args.args[0] is claimed + assert m_run.call_args.args[0]["fire_claim"]["by"] == "bg-owner" class TestInFlightDedupe: @@ -237,6 +251,136 @@ def probe_run(job, **kw): assert seen_during_run["registered"] is True assert "job-bg-09" not in sched.get_running_job_ids() # released after + def test_run_claimed_job_redacts_persisted_failure_detail(self): + from tools.cronjob_tools import _run_claimed_job + + sentinel = "RAW_PERSISTED_RUN_SENTINEL user@example.org /private/report.pdf" + with patch("cron.scheduler.run_one_job", return_value=False), \ + patch("tools.cronjob_tools.get_job", return_value={ + "last_status": "error", "last_error": sentinel, + }): + result = _run_claimed_job(_job("job-redacted-persisted")) + + assert result["success"] is False + assert result["error"] == "run_failed" + assert result["error_kind"] == "run_failed" + assert sentinel not in json.dumps(result, sort_keys=True) + + def test_run_claimed_job_redacts_exception_detail(self): + from tools.cronjob_tools import _run_claimed_job + + sentinel = "RAW_RUN_EXCEPTION_SENTINEL user@example.org /private/report.pdf" + with patch("cron.scheduler.run_one_job", side_effect=RuntimeError(sentinel)), \ + patch("tools.cronjob_tools.mark_job_run"): + result = _run_claimed_job(_job("job-redacted-exception")) + + assert result["success"] is False + assert result["error"] == "run_failed" + assert result["error_kind"] == "run_failed" + assert sentinel not in json.dumps(result, sort_keys=True) + + def test_execute_job_now_redacts_claim_exception_detail(self): + from tools.cronjob_tools import _execute_job_now + + sentinel = "RAW_CLAIM_EXCEPTION_SENTINEL user@example.org /private/report.pdf" + with patch( + "tools.cronjob_tools.claim_job_for_fire", + side_effect=RuntimeError(sentinel), + ), patch("tools.cronjob_tools.mark_job_run"): + result = _execute_job_now(_job("job-redacted-claim")) + + assert result["success"] is False + assert result["error"] == "run_failed" + assert result["error_kind"] == "run_failed" + assert sentinel not in json.dumps(result, sort_keys=True) + + def test_background_dispatch_redacts_claim_exception_detail(self): + from tools.cronjob_tools import _try_dispatch_background_run + + sentinel = "RAW_BACKGROUND_CLAIM_SENTINEL user@example.org /private/report.pdf" + with _bound_session_key(), patch( + "tools.cronjob_tools.claim_job_for_fire", + side_effect=RuntimeError(sentinel), + ), patch("tools.cronjob_tools.mark_job_run"): + result = _try_dispatch_background_run(_job("job-redacted-background")) + + assert result["success"] is False + assert result["error"] == "run_failed" + assert result["error_kind"] == "run_failed" + assert sentinel not in json.dumps(result, sort_keys=True) + + def test_cronjob_list_projection_is_summary_only(self): + private = { + "id": "bounded-tool-job", + "name": "bounded tool job", + "prompt": "PRIVATE_PROMPT user@example.org", + "script": "private/script.py", + "monitor_script": "private/monitor.py", + "monitor_url": "https://private.example.org/feed", + "workdir": "/private/worktree", + "model": "private-model", + "provider": "private-provider", + "base_url": "https://private-provider.example.org/v1", + "profile": "private-profile", + "context_from": ["private-upstream"], + "enabled_toolsets": ["private-toolset"], + "skills": ["private-skill"], + "reasoning_effort": "private-effort", + "monitor_state": {"private": "PRIVATE_MONITOR_STATE"}, + "schedule": {"kind": "cron", "expr": "0 9 * * *"}, + } + + public = _format_job(private) + serialized = json.dumps(public, sort_keys=True) + + assert public["job_id"] == "bounded-tool-job" + assert public["name"] == "bounded tool job" + for field in ( + "prompt_preview", "script", "monitor_script", "monitor_url", + "workdir", "model", "provider", "base_url", "profile", + "context_from", "enabled_toolsets", "skill", "skills", + "reasoning_effort", "monitor_state", + ): + assert field not in public + assert "PRIVATE_" not in serialized + assert "user@example.org" not in serialized + assert "/private/" not in serialized + + unnamed = _format_job({**private, "name": None}) + assert unnamed["name"] == "bounded-tool-job" + assert "PRIVATE_" not in json.dumps(unnamed, sort_keys=True) + + def test_cronjob_summary_projection_rejects_malformed_private_objects(self): + malformed = { + "id": "malformed-tool-job", + "name": {"private": "PRIVATE_NAME_SENTINEL"}, + "schedule_display": ["PRIVATE_SCHEDULE_SENTINEL"], + "repeat": ["PRIVATE_REPEAT_SENTINEL"], + "deliver": {"private": "PRIVATE_TARGET_SENTINEL"}, + "enabled": "PRIVATE_ENABLED_SENTINEL", + } + + public = _format_job(malformed) + serialized = json.dumps(public, sort_keys=True) + + assert public["job_id"] == "malformed-tool-job" + assert public["name"] == "malformed-tool-job" + assert public["schedule"] == "?" + assert public["repeat"] == "forever" + assert public["delivery_kind"] == "local" + assert public["enabled"] is True + assert "PRIVATE_" not in serialized + + def test_cronjob_outer_runtime_error_is_categorical(self): + sentinel = "RAW_OUTER_TOOL_SENTINEL user@example.org /private/report.pdf" + with patch("tools.cronjob_tools.list_jobs", side_effect=RuntimeError(sentinel)): + result = json.loads(cronjob(action="list")) + + assert result["success"] is False + assert result["error"] == "cron_operation_failed" + assert result["error_kind"] == "cron_operation_failed" + assert sentinel not in json.dumps(result, sort_keys=True) + def test_run_claimed_job_reports_exact_unknown_execution_not_stale_success(self): from tools.cronjob_tools import _run_claimed_job @@ -257,7 +401,9 @@ def probe_run(job, **_kwargs): res = _run_claimed_job(_job("job-bg-unknown")) assert res["success"] is False - assert res["error"] == "worker owner exited" + assert res["error"] == "run_failed" + assert res["error_kind"] == "run_failed" + assert "worker owner exited" not in str(res) def test_background_dispatch_reports_running_job_immediately(self): """The dispatch path pre-checks the running set so a mid-run job @@ -310,7 +456,7 @@ def test_run_action_returns_background_note(self): assert out["success"] is True assert out["job"]["executed"] is True assert out["job"]["execution_mode"] == "background" - assert out["job"]["delegation_id"] + assert "delegation_id" not in out["job"] assert "background" in out["note"] def test_run_action_sync_path_unchanged_without_session(self): diff --git a/tests/tools/test_cronjob_run_immediate.py b/tests/tools/test_cronjob_run_immediate.py index aa0eb8b97f76..8653d3834dfc 100644 --- a/tests/tools/test_cronjob_run_immediate.py +++ b/tests/tools/test_cronjob_run_immediate.py @@ -112,7 +112,8 @@ def test_run_reports_failure_from_last_status(self): assert out["job"]["executed"] is True assert out["job"]["execution_success"] is False - assert out["job"]["execution_error"] == "provider 500" + assert out["job"]["execution_error"] == "run_failed" + assert "provider 500" not in json.dumps(out) def test_execute_job_now_bails_without_claim(self): """_execute_job_now never calls run_one_job when the claim is lost.""" @@ -172,7 +173,7 @@ def test_execute_job_now_marks_failure_on_exception(self): res = _execute_job_now(dict(_JOB)) assert res["claimed"] is True assert res["success"] is False - assert "boom" in res["error"] + assert res["error"] == "run_failed" m_mark.assert_called_once_with( "job-run-1", False, @@ -313,7 +314,9 @@ def test_delivery_failed_status_is_not_success_and_surfaces_reason(self): assert res["claimed"] is True assert res["success"] is False - assert "502" in res["error"] + assert res["error"] == "run_failed" + assert res["error_kind"] == "run_failed" + assert "502" not in str(res) def test_plain_ok_is_still_success_with_no_error(self): with patch("tools.cronjob_tools.claim_job_for_fire", diff --git a/tests/tools/test_cronjob_tools.py b/tests/tools/test_cronjob_tools.py index 12aa49c80aae..497e39cafda5 100644 --- a/tests/tools/test_cronjob_tools.py +++ b/tests/tools/test_cronjob_tools.py @@ -246,6 +246,29 @@ def test_create_and_list(self): assert listing["jobs"][0]["name"] == "Server Check" assert listing["jobs"][0]["state"] == "scheduled" + def test_create_response_redacts_delivery_target_and_execution_config(self): + created = json.loads( + cronjob( + action="create", + prompt="Private execution prompt", + schedule="every 1h", + name="Bounded label", + deliver="telegram:-1001234567890:98765", + skills=["private-skill"], + ) + ) + + assert created["success"] is True + assert created["delivery_kind"] == "external" + assert "deliver" not in created + assert "skill" not in created + assert "skills" not in created + assert created["job"]["delivery_kind"] == "external" + serialized = json.dumps(created, sort_keys=True) + assert "-1001234567890" not in serialized + assert "private-skill" not in serialized + assert "Private execution prompt" not in serialized + def test_create_with_natural_weekday_schedule(self): # The documented "every monday 9am" form must create a real cron job # through the tool path, not error out (issue: parser rejected it). @@ -303,7 +326,7 @@ def test_list_handles_partial_legacy_job_records(self): assert listing["success"] is True assert listing["jobs"][0]["name"] == "abc123deadbe" - assert listing["jobs"][0]["prompt_preview"] == "" + assert "prompt_preview" not in listing["jobs"][0] assert listing["jobs"][0]["schedule"] == "every 60m" def test_pause_and_resume(self): @@ -372,6 +395,8 @@ def test_legacy_unsafe_job_blocked_on_unrelated_update(self, monkeypatch): def test_legacy_unsafe_job_remediated_by_matching_host(self, monkeypatch): """Repointing base_url at the named provider's own configured host also remediates the job (no off-host exfil).""" + from cron.jobs import get_job + self._patch_named_legit(monkeypatch) job_id = self._save_legacy_unsafe_job() @@ -380,7 +405,8 @@ def test_legacy_unsafe_job_remediated_by_matching_host(self, monkeypatch): base_url="https://legit.example/v1") ) assert result["success"] is True - assert result["job"]["base_url"] == "https://legit.example/v1" + assert "base_url" not in result["job"] + assert get_job(job_id)["base_url"] == "https://legit.example/v1" def test_create_normalizes_list_form_deliver(self): @@ -573,6 +599,57 @@ def test_update_persists_attach_to_session(self): listed = next(j for j in listing["jobs"] if j["job_id"] == created["job_id"]) assert listed.get("attach_to_session") is False + @pytest.mark.parametrize("invalid", ["false", 0, 1, [], {}]) + def test_create_rejects_non_boolean_attach_to_session(self, invalid): + from cron.jobs import list_jobs + from tools.registry import registry + + result = json.loads( + registry.dispatch( + "cronjob_manage", + { + "action": "create", + "schedule": "1h", + "prompt": "fire and forget", + "attach_to_session": invalid, + }, + ) + ) + + assert result["success"] is False + assert "boolean" in result["error"].lower() + assert list_jobs(include_disabled=True) == [] + + @pytest.mark.parametrize("invalid", ["false", 0, 1, [], {}]) + def test_update_rejects_non_boolean_attach_to_session(self, invalid): + from cron.jobs import get_job + from tools.registry import registry + + created = json.loads( + registry.dispatch( + "cronjob_manage", + { + "action": "create", + "schedule": "1h", + "prompt": "fire and forget", + }, + ) + ) + result = json.loads( + registry.dispatch( + "cronjob_manage", + { + "action": "update", + "job_id": created["job_id"], + "attach_to_session": invalid, + }, + ) + ) + + assert result["success"] is False + assert "boolean" in result["error"].lower() + assert "attach_to_session" not in (get_job(created["job_id"]) or {}) + def test_omitted_create_leaves_field_absent(self): from cron.jobs import get_job from tools.registry import registry @@ -628,7 +705,7 @@ def test_omitted_deliver_no_origin_emits_notice(self): ) assert created["success"] is True # Omitted deliver from a session with no origin downgrades to local. - assert created["deliver"] == "local" + assert created["delivery_kind"] == "local" assert "local-only cron job" in created["message"] assert "deliver='telegram'" in created["message"] @@ -642,7 +719,7 @@ def test_gateway_origin_no_notice(self, monkeypatch): created = json.loads( cronjob(action="create", prompt="x", schedule="every 2m") ) - assert created["deliver"] == "origin" + assert created["delivery_kind"] == "origin" assert "local-only cron job" not in created["message"] diff --git a/tests/tools/test_telegram_send_message_caption.py b/tests/tools/test_telegram_send_message_caption.py index 34240701ab03..f77cd496d093 100644 --- a/tests/tools/test_telegram_send_message_caption.py +++ b/tests/tools/test_telegram_send_message_caption.py @@ -105,3 +105,24 @@ def test_multi_file_keeps_separate_text(monkeypatch: pytest.MonkeyPatch) -> None finally: os.unlink(img) os.unlink(img2) + + +def test_receipt_bound_missing_caption_media_does_not_send_unplanned_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from tools.send_message_tool import _send_telegram + + _no_proxy(monkeypatch) + bot = _make_bot() + _install_telegram_mock(monkeypatch, MagicMock(return_value=bot)) + + res = asyncio.run(_send_telegram( + "tok", "123", "planned caption", + media_files=[("/missing/bounded-image.png", False)], + receipt_bound=True, + )) + + assert "error" in res + assert not res.get("receipts") + bot.send_message.assert_not_awaited() + bot.send_photo.assert_not_awaited() diff --git a/tests/tools/test_transport_receipt_preservation.py b/tests/tools/test_transport_receipt_preservation.py new file mode 100644 index 000000000000..c78cd9cf5071 --- /dev/null +++ b/tests/tools/test_transport_receipt_preservation.py @@ -0,0 +1,421 @@ +"""Receipt preservation and redaction at tool-facing seams.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from gateway.config import Platform +from gateway.platforms.base import SendResult, TransportReceipt, TransportTarget + + +@pytest.mark.asyncio +async def test_send_via_adapter_preserves_partial_receipts_on_failure(): + from tools.send_message_tool import _send_via_adapter + + target = TransportTarget("matrix", "!room:example.org") + receipt = TransportReceipt( + outcome="delivered", provider_message_id="$event-1", + requested_target=target, actual_target=target, + component="text", ordinal=0, + ) + adapter = SimpleNamespace(send=AsyncMock(return_value=SendResult( + success=False, error="second chunk rejected", receipts=(receipt,), + ))) + runner = SimpleNamespace(adapters={Platform.MATRIX: adapter}) + + with patch("gateway.run._gateway_runner_ref", return_value=runner): + result = await _send_via_adapter( + Platform.MATRIX, SimpleNamespace(extra={}), + "!room:example.org", "message", + ) + + assert result["receipts"] == (receipt,) + assert "second chunk rejected" in result["error"] + + +@pytest.mark.asyncio +async def test_standalone_telegram_preserves_every_text_ack_and_target(): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + messages = [SimpleNamespace(message_id=101), SimpleNamespace(message_id=102)] + with ( + patch("telegram.Bot", return_value=object()), + patch("plugins.platforms.telegram.adapter.TelegramAdapter.format_message", return_value="formatted"), + patch("gateway.platforms.base.BasePlatformAdapter.truncate_message", return_value=["one", "two"]), + patch("tools.send_message_tool._send_telegram_message_with_retry", new=AsyncMock(side_effect=messages)), + ): + result = await _send_telegram("test-token", "-100123", "report", thread_id="7") + + assert result["success"] is True + assert [receipt.provider_message_id for receipt in result["receipts"]] == ["101", "102"] + assert [receipt.ordinal for receipt in result["receipts"]] == [0, 1] + assert all(receipt.requested_target.thread_id == "7" for receipt in result["receipts"]) + assert all(receipt.actual_target.thread_id == "7" for receipt in result["receipts"]) + + +@pytest.mark.asyncio +async def test_standalone_telegram_text_ack_uses_inert_provider_id_normalization(): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + string_calls = [] + + class HostileProviderId(str): + def __str__(self): + string_calls.append("called") + return "provider-id" + + sender = AsyncMock( + return_value=SimpleNamespace(message_id=HostileProviderId("provider-id")) + ) + with ( + patch("telegram.Bot", return_value=object()), + patch( + "plugins.platforms.telegram.adapter.TelegramAdapter.format_message", + return_value="formatted", + ), + patch( + "gateway.platforms.base.BasePlatformAdapter.truncate_message", + return_value=["one"], + ), + patch( + "tools.send_message_tool._send_telegram_message_with_retry", + new=sender, + ), + ): + result = await _send_telegram( + "test-token", "-100123", "report", receipt_bound=True, + ) + + assert result["success"] is True + assert result["message_id"] == "provider-id" + assert len(result["receipts"]) == 1 + assert result["receipts"][0].outcome == "delivered" + assert result["receipts"][0].provider_message_id == "provider-id" + assert sender.await_count == 1 + assert string_calls == [] + + +@pytest.mark.asyncio +async def test_standalone_telegram_invalid_text_ack_is_unknown_without_magic(): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + magic_calls = [] + + class HostileProviderId: + @property + def __class__(self): + magic_calls.append("class") + return str + + def __str__(self): + magic_calls.append("str") + return "spoofed-provider-id" + + sender = AsyncMock( + return_value=SimpleNamespace(message_id=HostileProviderId()) + ) + with ( + patch("telegram.Bot", return_value=object()), + patch( + "plugins.platforms.telegram.adapter.TelegramAdapter.format_message", + return_value="formatted", + ), + patch( + "gateway.platforms.base.BasePlatformAdapter.truncate_message", + return_value=["one"], + ), + patch( + "tools.send_message_tool._send_telegram_message_with_retry", + new=sender, + ), + ): + result = await _send_telegram( + "test-token", "-100123", "report", receipt_bound=True, + ) + + assert result["error_kind"] == "unknown" + assert result["retryable"] is False + assert len(result["receipts"]) == 1 + assert result["receipts"][0].outcome == "unknown" + assert result["receipts"][0].provider_message_id is None + assert sender.await_count == 1 + assert magic_calls == [] + + +@pytest.mark.asyncio +async def test_receipt_bound_standalone_telegram_does_not_plaintext_retry_parse_error(): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + sender = AsyncMock(side_effect=RuntimeError("parse entities rejected")) + with ( + patch("telegram.Bot", return_value=object()), + patch( + "plugins.platforms.telegram.adapter.TelegramAdapter.format_message", + return_value="formatted", + ), + patch( + "gateway.platforms.base.BasePlatformAdapter.truncate_message", + return_value=["formatted"], + ), + patch( + "tools.send_message_tool._send_telegram_message_with_retry", + new=sender, + ), + ): + result = await _send_telegram( + "test-token", "-100123", "report", receipt_bound=True, + ) + + assert "error" in result + assert sender.await_count == 1 + + +@pytest.mark.asyncio +async def test_standalone_telegram_thread_fallback_keeps_requested_target_truthful(): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + with ( + patch("telegram.Bot", return_value=object()), + patch("plugins.platforms.telegram.adapter.TelegramAdapter.format_message", return_value="formatted"), + patch("gateway.platforms.base.BasePlatformAdapter.truncate_message", return_value=["one"]), + patch( + "tools.send_message_tool._send_telegram_message_with_retry", + new=AsyncMock(side_effect=[ + RuntimeError("Message thread not found"), + SimpleNamespace(message_id=103), + ]), + ), + ): + result = await _send_telegram("test-token", "-100123", "report", thread_id="7") + + receipt = result["receipts"][0] + assert receipt.requested_target.thread_id == "7" + assert receipt.actual_target.thread_id is None + + +@pytest.mark.asyncio +async def test_standalone_telegram_preserves_media_provider_ack(tmp_path): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + image = tmp_path / "image.jpg" + image.write_bytes(b"bounded-image") + bot = SimpleNamespace( + send_photo=AsyncMock(return_value=SimpleNamespace(message_id=201)), + ) + with patch("telegram.Bot", return_value=bot): + result = await _send_telegram( + "test-token", "-100123", "", thread_id="7", + media_files=[(str(image), False)], + ) + + receipt = result["receipts"][0] + assert receipt.provider_message_id == "201" + assert receipt.component == "media" + assert receipt.ordinal == 0 + assert receipt.requested_target.thread_id == "7" + assert receipt.actual_target.thread_id == "7" + + +@pytest.mark.asyncio +async def test_standalone_telegram_media_ack_uses_inert_provider_id_normalization( + tmp_path, +): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + string_calls = [] + + class HostileProviderId(str): + def __str__(self): + string_calls.append("called") + return "media-provider-id" + + image = tmp_path / "image.jpg" + image.write_bytes(b"bounded-image") + sender = AsyncMock( + return_value=SimpleNamespace( + message_id=HostileProviderId("media-provider-id") + ) + ) + bot = SimpleNamespace(send_photo=sender) + with patch("telegram.Bot", return_value=bot): + result = await _send_telegram( + "test-token", "-100123", "", thread_id="7", + media_files=[(str(image), False)], receipt_bound=True, + ) + + assert result["success"] is True + assert result["message_id"] == "media-provider-id" + assert len(result["receipts"]) == 1 + assert result["receipts"][0].outcome == "delivered" + assert result["receipts"][0].provider_message_id == "media-provider-id" + assert sender.await_count == 1 + assert string_calls == [] + + +@pytest.mark.asyncio +async def test_standalone_telegram_missing_media_caption_fallback_normalizes_ack( + tmp_path, +): + pytest.importorskip("telegram") + from tools.send_message_tool import _send_telegram + from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: F401 + + sender = AsyncMock(return_value=SimpleNamespace(message_id=301)) + bot = SimpleNamespace(send_message=sender) + with patch("telegram.Bot", return_value=bot): + result = await _send_telegram( + "test-token", "-100123", "bounded caption", + media_files=[(str(tmp_path / "missing.jpg"), False)], + ) + + assert result["success"] is True + assert result["message_id"] == "301" + assert sender.await_count == 1 + + +@pytest.mark.asyncio +async def test_matrix_send_core_preserves_typed_receipts_on_success_and_failure(): + from tools.send_message_tool import _matrix_send_core + + target = TransportTarget("matrix", "!room:example.org") + receipts = tuple( + TransportReceipt( + outcome="delivered", provider_message_id=f"$event-{ordinal}", + requested_target=target, actual_target=target, + component="text", ordinal=ordinal, + ) + for ordinal in range(2) + ) + adapter = SimpleNamespace(send=AsyncMock(return_value=SendResult( + success=True, message_id="$event-1", receipts=receipts, + ))) + success = await _matrix_send_core(adapter, target.chat_id, "report", [], None) + assert success["receipts"] == receipts + + adapter.send = AsyncMock(return_value=SendResult( + success=False, error="second chunk failed", receipts=(receipts[0],), + )) + failed = await _matrix_send_core(adapter, target.chat_id, "report", [], None) + assert failed["receipts"] == (receipts[0],) + + +def test_cron_tool_execution_surface_is_bounded_and_redacted(monkeypatch, tmp_path): + import cron.executions as executions + from tools.cronjob_tools import _format_job + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db") + execution = executions.create_execution("tool-job", source="builtin") + executions.finish_execution( + execution["id"], success=False, + error="RAW_TOOL_ERROR_SENTINEL user@example.org /private/report.pdf", + ) + + formatted = _format_job({ + "id": "tool-job", "name": "tool job", "prompt": "normal prompt", + "schedule_display": "every 1h", "enabled": True, + "last_status": "error", + "last_error": "RAW_LAST_ERROR_SENTINEL user@example.org", + "last_delivery_error": "RAW_DELIVERY_SENTINEL /private/report.pdf", + "last_fire_error": { + "at": "2026-08-22T20:00:00+00:00", + "detail": "RAW_FIRE_SENTINEL provider payload", + }, + "last_dispatch": { + "scheduled_at": "2026-09-01T09:00:00+00:00", + "dispatched_at": "2026-09-01T09:31:00+00:00", + "lateness_seconds": 1860, + "kind": "catch_up", + "detail": "RAW_DISPATCH_SENTINEL provider payload", + }, + }) + serialized = json.dumps(formatted, sort_keys=True) + + assert formatted["last_execution"] == { + "status": "failed", + "receipt": { + "delivered": 0, "failed": 0, "unknown": 0, + "targets_delivered": 0, + }, + } + assert formatted["last_delivery_error"] == "delivery_failed" + assert formatted["last_fire_error"] == { + "at": "2026-08-22T20:00:00+00:00", + "error_kind": "fire_forward_failed", + } + assert formatted["last_dispatch"] == { + "scheduled_at": "2026-09-01T09:00:00+00:00", + "dispatched_at": "2026-09-01T09:31:00+00:00", + "lateness_seconds": 1860.0, + "kind": "catch_up", + } + assert "RAW_TOOL_ERROR_SENTINEL" not in serialized + assert "RAW_LAST_ERROR_SENTINEL" not in serialized + assert "RAW_DELIVERY_SENTINEL" not in serialized + assert "RAW_FIRE_SENTINEL" not in serialized + assert "RAW_DISPATCH_SENTINEL" not in serialized + assert "user@example.org" not in serialized + assert "/private/report.pdf" not in serialized + + +def test_cron_tool_drops_malformed_fire_timestamp(): + from tools.cronjob_tools import _format_job + + sentinel = "RAW_TOOL_FIRE_AT_SENTINEL user@example.org /private/report.pdf" + formatted = _format_job({ + "id": "malformed-fire-at", + "name": "malformed fire", + "enabled": True, + "last_fire_error": {"at": sentinel, "detail": "raw detail"}, + }) + + assert formatted["last_fire_error"] == { + "at": None, + "error_kind": "fire_forward_failed", + } + assert sentinel not in json.dumps(formatted, sort_keys=True) + + +def test_cron_tool_public_projection_rejects_subclasses_before_magic_methods(): + from tools.cronjob_tools import _format_job + + class HostileDict(dict): + def get(self, *_args, **_kwargs): + raise AssertionError("hostile job get was called") + + class HostileText(str): + def __bool__(self): + raise AssertionError("hostile text truthiness was evaluated") + + def __len__(self): + raise AssertionError("hostile text length was evaluated") + + with pytest.raises(TypeError, match="object"): + _format_job(HostileDict({"id": "hostile"})) + + public = _format_job({ + "id": "bounded", + "name": HostileText("private"), + "deliver": HostileText("external:private"), + "last_fire_error": HostileDict({"at": "private"}), + }) + assert public["name"] == "bounded" + assert public["delivery_kind"] == "local" + assert public["last_fire_error"] is None diff --git a/tools/cronjob_tools.py b/tools/cronjob_tools.py index e59f364979f2..5b1bc4fa0b7f 100644 --- a/tools/cronjob_tools.py +++ b/tools/cronjob_tools.py @@ -7,10 +7,12 @@ import json import logging +import math import re import sys import threading import time +from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -39,6 +41,7 @@ from cron.jobs import ( AmbiguousJobReference, + EMPTY_PAYLOAD_ERROR, claim_job_for_fire, effective_job_state, get_job, @@ -492,10 +495,17 @@ def _split_monitor_arg( def _repeat_display(job: Dict[str, Any]) -> str: - times = (job.get("repeat") or {}).get("times") - completed = (job.get("repeat") or {}).get("completed", 0) + repeat = job.get("repeat") + if type(repeat) is not dict: + repeat = {} + times = repeat.get("times") + completed = repeat.get("completed", 0) if times is None: return "forever" + if type(times) is not int or times < 1: + return "forever" + if type(completed) is not int or completed < 0: + completed = 0 if times == 1: return "once" if completed == 0 else "1/1" return f"{completed}/{times}" if completed else f"{times} times" @@ -749,64 +759,134 @@ def _validate_cron_script_path(script: Optional[str]) -> Optional[str]: return None +def _public_timestamp(value: Any) -> Optional[str]: + if type(value) is not str or len(value) > 64: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None or parsed.utcoffset() is None: + return None + return value + + +def _public_tool_text(value: Any, *, limit: int) -> Optional[str]: + if type(value) is not str or not value or len(value) > limit: + return None + return value if all(char.isprintable() for char in value) else None + + +def _public_tool_category(value: Any) -> Optional[str]: + if type(value) is not str: + return None + return value if re.fullmatch(r"[a-z][a-z0-9_]{0,31}", value) else None + + +def _public_last_dispatch(value: Any) -> Optional[Dict[str, Any]]: + """Project one bounded, content-free scheduler dispatch stamp.""" + if type(value) is not dict: + return None + scheduled_at = _public_timestamp(value.get("scheduled_at")) + dispatched_at = _public_timestamp(value.get("dispatched_at")) + kind = value.get("kind") + lateness = value.get("lateness_seconds") + if ( + scheduled_at is None + or dispatched_at is None + or kind not in {"on_time", "catch_up", "late"} + or type(lateness) not in {int, float} + ): + return None + assert isinstance(lateness, (int, float)) + lateness_value = float(lateness) + if not math.isfinite(lateness_value) or lateness_value < 0: + return None + return { + "scheduled_at": scheduled_at, + "dispatched_at": dispatched_at, + "lateness_seconds": lateness_value, + "kind": kind, + } + + def _format_job(job: Dict[str, Any]) -> Dict[str, Any]: - prompt = str(job.get("prompt") or "") - skills = _canonical_skills(job.get("skill"), job.get("skills")) - job_id = str(job.get("id") or "unknown") - name = str(job.get("name") or prompt[:50] or (skills[0] if skills else "") or job_id or "cron job") + if type(job) is not dict: + raise TypeError("cron job projection requires an object") + raw_id = job.get("id") + job_id = ( + raw_id + if type(raw_id) is str + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}", raw_id) + else "unknown" + ) + name = _public_tool_text(job.get("name"), limit=256) or job_id + schedule = _public_tool_text(job.get("schedule_display"), limit=256) or "?" + deliver = job.get("deliver") + if type(deliver) is str and deliver in {"local", "origin", "all"}: + delivery_kind = deliver + elif type(deliver) is str and deliver: + delivery_kind = "external" + else: + delivery_kind = "local" + if any( + type(job.get(key)) is str and bool(job.get(key)) + for key in ("monitor_script", "monitor_url") + ): + mode = "monitor" + elif job.get("no_agent") is True: + mode = "script" + else: + mode = "agent" result = { "job_id": job_id, "name": name, - "skill": skills[0] if skills else None, - "skills": skills, - "prompt_preview": prompt[:100] + "..." if len(prompt) > 100 else prompt, - "model": job.get("model"), - "provider": job.get("provider"), - "base_url": job.get("base_url"), - "schedule": job.get("schedule_display") or "?", + "schedule": schedule, "repeat": _repeat_display(job), - "deliver": job.get("deliver", "local"), - "next_run_at": job.get("next_run_at"), - "last_run_at": job.get("last_run_at"), - "last_status": job.get("last_status"), - "last_delivery_error": job.get("last_delivery_error"), - "last_delivery_unverified": job.get("last_delivery_unverified"), - "last_fire_error": job.get("last_fire_error"), - "enabled": job.get("enabled", True), - # Derive from enabled so half-paused records never render as paused. - "state": effective_job_state(job), - "paused_at": job.get("paused_at"), - "paused_reason": job.get("paused_reason"), + "delivery_kind": delivery_kind, + "mode": mode, + "next_run_at": _public_timestamp(job.get("next_run_at")), + "last_run_at": _public_timestamp(job.get("last_run_at")), + "last_dispatch": _public_last_dispatch(job.get("last_dispatch")), + "last_status": _public_tool_category(job.get("last_status")), + "last_delivery_error": ( + "delivery_failed" if job.get("last_delivery_error") is not None else None + ), + "last_delivery_unverified": ( + True + if type(job.get("last_delivery_unverified")) is list + and len(job["last_delivery_unverified"]) > 0 + else None + ), + "last_fire_error": ( + { + "at": _public_timestamp(job["last_fire_error"].get("at")), + "error_kind": "fire_forward_failed", + } + if type(job.get("last_fire_error")) is dict + else None + ), + "enabled": job.get("enabled") if type(job.get("enabled")) is bool else True, + "state": _public_tool_category(effective_job_state(job)), } - if job.get("script"): - result["script"] = job["script"] - if job.get("reasoning_effort"): - result["reasoning_effort"] = job["reasoning_effort"] - if job.get("monitor_script"): - result["monitor_script"] = job["monitor_script"] - if job.get("monitor_url"): - result["monitor_url"] = job["monitor_url"] - if job.get("monitor_state"): - result["monitor_state"] = job["monitor_state"] - if job.get("no_agent"): - result["no_agent"] = True - if job.get("enabled_toolsets"): - result["enabled_toolsets"] = job["enabled_toolsets"] - if job.get("workdir"): - result["workdir"] = job["workdir"] - stored_refs = job.get("context_from") or [] - if isinstance(stored_refs, str): - stored_refs = [stored_refs] - if any(str(r).strip().lower() == "self" or r == job.get("id") for r in stored_refs): - result["continuity"] = True - external_refs = [ - r for r in stored_refs - if str(r).strip().lower() != "self" and r != job.get("id") - ] - if external_refs: - result["context_from"] = external_refs if isinstance(job.get("attach_to_session"), bool): + # Boolean conversation-continuity intent is safe to project; keep + # target IDs, prompts, scripts, skills, workdirs and provider config + # bounded out of the public tool result. result["attach_to_session"] = job["attach_to_session"] + try: + from cron.executions import latest_execution, receipt_summary + + execution = latest_execution(job_id) + if execution is not None: + result["last_execution"] = { + "status": _public_tool_category(execution.get("status")), + "receipt": receipt_summary(execution["id"]), + } + except Exception: + # Listing cron jobs must stay available if the optional audit DB is + # unavailable; do not surface raw SQLite/provider details to the tool. + pass return result @@ -961,7 +1041,7 @@ def _execute_job_now( failure delivery, ``[SILENT]`` handling, and live-adapter delivery stay identical across paths and can't drift. - Returns {"claimed": bool, "success": bool, "error": str|None}. + Returns a bounded public result; raw execution details remain internal. """ job_id = job["id"] claimed_job = None @@ -987,7 +1067,12 @@ def _execute_job_now( mark_job_run(job_id, False, str(e)) except Exception: pass - return {"claimed": True, "success": False, "error": str(e)} + return { + "claimed": True, + "success": False, + "error": "run_failed", + "error_kind": "run_failed", + } return _run_claimed_job(claimed_job, extra_prompt=extra_prompt) @@ -1002,7 +1087,7 @@ def _run_claimed_job( the tool response can report "paused"/"already firing" immediately — and hand the actual run to a daemon worker. - Returns {"claimed": True, "success": bool, "error": str|None}. + Returns a bounded public result; raw execution details remain internal. """ job_id = job["id"] _registered = False @@ -1151,7 +1236,8 @@ def _heartbeat_loop() -> None: return { "claimed": True, "success": bool(processed and ok), - "error": run_error, + "error": None if processed and ok else "run_failed", + "error_kind": None if processed and ok else "run_failed", } except Exception as e: @@ -1179,7 +1265,8 @@ def _heartbeat_loop() -> None: return { "claimed": True, "success": False, - "error": str(e), + "error": "run_failed", + "error_kind": "run_failed", } @@ -1347,7 +1434,13 @@ def _try_dispatch_background_run( mark_job_run(job_id, False, str(e)) except Exception: pass - return {"claimed": True, "dispatched": False, "success": False, "error": str(e)} + return { + "claimed": True, + "dispatched": False, + "success": False, + "error": "run_failed", + "error_kind": "run_failed", + } origin_ui_session_id = "" try: @@ -1445,7 +1538,7 @@ def _runner() -> Dict[str, Any]: "cronjob run: background pool unavailable (%s); running job '%s' inline.", dispatch.get("error", "rejected"), job_name, ) - result = _run_claimed_job(job, extra_prompt=extra_prompt) + result = _run_claimed_job(claimed_job, extra_prompt=extra_prompt) result["dispatched"] = False return result @@ -1541,6 +1634,12 @@ def cronjob( try: normalized = (action or "").strip().lower() + if attach_to_session is not None and type(attach_to_session) is not bool: + return tool_error( + "attach_to_session must be a boolean.", + success=False, + error_kind="invalid_argument", + ) if normalized == "create": if not schedule: @@ -1662,26 +1761,22 @@ def cronjob( except CronSchedulerRegistrationError as exc: _partial = exc.to_dict() return tool_error(_partial.pop("error"), success=False, **_partial) - _create_message = f"Cron job '{job['name']}' created." + public_job = _format_job(job) + _create_message = f"Cron job '{public_job['name']}' created." _local_notice = _local_delivery_notice(job, _normalize_deliver_param(deliver)) if _local_notice: _create_message = f"{_create_message} {_local_notice}" - # Gateway liveness surfacing (#87033): the builtin scheduler's - # ticker lives in the gateway process, so a job created with no - # gateway running is stored but will never fire. Tell the model - # here — the CLI already warns, but the agent path saw only a - # clean success and confidently told the user it was scheduled. + # Keep the PR's bounded public projection while preserving the + # upstream gateway-liveness warning for newly created jobs. _result = { "success": True, - "job_id": job["id"], - "name": job["name"], - "skill": job.get("skill"), - "skills": job.get("skills", []), - "schedule": job["schedule_display"], - "repeat": _repeat_display(job), - "deliver": job.get("deliver", "local"), - "next_run_at": job["next_run_at"], - "job": _format_job(job), + "job_id": public_job["job_id"], + "name": public_job["name"], + "schedule": public_job["schedule"], + "repeat": public_job["repeat"], + "delivery_kind": public_job["delivery_kind"], + "next_run_at": public_job["next_run_at"], + "job": public_job, "message": _create_message, **_gateway_liveness_notice(), } @@ -1795,7 +1890,6 @@ def cronjob( result = _format_job(get_job(job_id) or {"id": job_id}) result["executed"] = True result["execution_mode"] = "background" - result["delegation_id"] = bg.get("delegation_id") return json.dumps( { "success": True, @@ -1970,7 +2064,7 @@ def cronjob( if enabled_toolsets is not None: updates["enabled_toolsets"] = enabled_toolsets or None if attach_to_session is not None: - updates["attach_to_session"] = bool(attach_to_session) + updates["attach_to_session"] = attach_to_session if workdir is not None: # Empty string clears the field (restores old behaviour); # otherwise pass raw — update_job() validates / normalizes. @@ -2021,7 +2115,23 @@ def cronjob( return tool_error(f"Unknown cron action '{action}'", success=False) except Exception as e: - return tool_error(str(e), success=False) + if isinstance(e, ValueError) and str(e) == EMPTY_PAYLOAD_ERROR: + return tool_error( + EMPTY_PAYLOAD_ERROR, + success=False, + error_kind="invalid_job_payload", + ) + if isinstance(e, ValueError) and "past and cannot be scheduled" in str(e): + return tool_error( + "One-shot schedule is in the past and cannot be scheduled.", + success=False, + error_kind="invalid_schedule", + ) + return tool_error( + "cron_operation_failed", + success=False, + error_kind="cron_operation_failed", + ) diff --git a/tools/send_message_tool.py b/tools/send_message_tool.py index 11a7bf40c2ac..fcf090439189 100644 --- a/tools/send_message_tool.py +++ b/tools/send_message_tool.py @@ -1061,8 +1061,24 @@ def _media_coro(): except Exception as e: return {"error": f"Plugin platform send failed: {_bounded_send_error(e)}"} if result.success: - return {"success": True, "message_id": result.message_id} - return {"error": f"Adapter send failed: {_bounded_send_error(result.error)}"} + # Receipts are content-free immutable evidence. Keep them on + # this internal result so cron can bind them to pre-registered + # attempts; legacy message_id remains only a display field. + normalized = { + "success": True, + "message_id": result.message_id, + } + receipts = tuple(getattr(result, "receipts", ()) or ()) + if receipts: + normalized["receipts"] = receipts + return normalized + normalized = { + "error": f"Adapter send failed: {_bounded_send_error(result.error)}" + } + receipts = tuple(getattr(result, "receipts", ()) or ()) + if receipts: + normalized["receipts"] = receipts + return normalized entry = None try: @@ -1109,7 +1125,10 @@ def _media_coro(): } -async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None, media_files=None, force_document=False, args=None): +async def _send_to_platform( + platform, pconfig, chat_id, message, thread_id=None, media_files=None, + force_document=False, args=None, receipt_bound=False, +): """Route a message to the appropriate platform sender. Long messages are automatically chunked to fit within platform limits @@ -1203,6 +1222,7 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None, thread_id=thread_id, disable_link_previews=disable_link_previews, force_document=force_document, + receipt_bound=receipt_bound, ) # --- Discord: chunked delivery via the registry's standalone_sender_fn. @@ -1556,7 +1576,43 @@ def _is_telegram_thread_not_found(error: Exception) -> bool: return "thread not found" in str(error).lower() -async def _send_telegram(token, chat_id, message, media_files=None, thread_id=None, disable_link_previews=False, force_document=False): +def _plan_standalone_telegram_text( + message: str, media_files=None, +) -> tuple[str, list[str], bool, str | None]: + """Format and split the exact text bytes used by standalone Telegram.""" + from gateway.platforms.base import BasePlatformAdapter, utf16_len + + has_html = bool(re.search(r'<[a-zA-Z/][^>]*>', message)) + if has_html: + formatted = message + else: + try: + from plugins.platforms.telegram.adapter import TelegramAdapter + adapter = TelegramAdapter.__new__(TelegramAdapter) + formatted = adapter.format_message(message) + except Exception: + formatted = message + caption = None + candidate_caption, _ = _media_caption_split( + message, media_files, max_caption_len=_TELEGRAM_CAPTION_LIMIT, + ) + if ( + candidate_caption is not None + and utf16_len(formatted) <= _TELEGRAM_CAPTION_LIMIT + ): + caption = formatted + chunks = ( + BasePlatformAdapter.truncate_message(formatted, 4096, len_fn=utf16_len) + if formatted.strip() and caption is None + else [] + ) + return formatted, list(chunks), has_html, caption + + +async def _send_telegram( + token, chat_id, message, media_files=None, thread_id=None, + disable_link_previews=False, force_document=False, receipt_bound=False, +): """Send via Telegram Bot API (one-shot, no polling needed). Applies markdown→MarkdownV2 formatting (same as the gateway adapter) @@ -1568,23 +1624,10 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No from telegram import Bot from telegram.constants import ParseMode - # Auto-detect HTML tags — if present, skip MarkdownV2 and send as HTML. - # Inspired by github.com/ashaney — PR #1568. - _has_html = bool(re.search(r'<[a-zA-Z/][^>]*>', message)) - - if _has_html: - formatted = message - send_parse_mode = ParseMode.HTML - else: - # Reuse the gateway adapter's format_message for markdown→MarkdownV2 - try: - from plugins.platforms.telegram.adapter import TelegramAdapter - _adapter = TelegramAdapter.__new__(TelegramAdapter) - formatted = _adapter.format_message(message) - except Exception: - # Fallback: send as-is if formatting unavailable - formatted = message - send_parse_mode = ParseMode.MARKDOWN_V2 + formatted, planned_text_chunks, _has_html, _tg_caption = ( + _plan_standalone_telegram_text(message, media_files=media_files) + ) + send_parse_mode = ParseMode.HTML if _has_html else ParseMode.MARKDOWN_V2 # Honour a configured proxy (telegram.proxy_url in config.yaml, exported # as TELEGRAM_PROXY env var by load_gateway_config). Without this, the @@ -1648,8 +1691,22 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No if disable_link_previews: text_kwargs["disable_web_page_preview"] = True + from gateway.platforms.base import ( + TransportReceipt, + TransportTarget, + normalize_transport_provider_message_id, + ) + last_msg = None + last_provider_message_id = None warnings = [] + receipts = [] + requested_thread = ( + str(effective_thread_id) + if thread_id is not None and effective_thread_id is not None + else None + ) + requested_target = TransportTarget("telegram", str(chat_id), requested_thread) # MEDIA: caption: when a single captionable file is accompanied # by short text, attach the text to the media bubble as its native @@ -1659,28 +1716,11 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No # the formatted length against Telegram's 1024 cap — formatting can # inflate a raw-<1024 string past it, in which case fall back to a # separate body message. - _tg_caption = None - from gateway.platforms.base import utf16_len as _utf16_len - _cap, _ = _media_caption_split( - message, media_files, max_caption_len=_TELEGRAM_CAPTION_LIMIT - ) - if _cap is not None and _utf16_len(formatted) <= _TELEGRAM_CAPTION_LIMIT: - _tg_caption = formatted - formatted = "" # suppress the separate text send below - + # The shared planner already selected caption-vs-text using the exact + # formatted UTF-16 length. ``_tg_caption`` is therefore also the + # preregistration decision for receipt-bound standalone sends. if formatted.strip(): - # Chunk *after* formatting: MarkdownV2/HTML escaping inflates the - # text (each escaped char like `!`/`.`/`-` becomes `\!`/`\.`/`\-`), - # so a message that fit under 4096 UTF-16 units raw can exceed the - # Telegram limit once formatted and get rejected as "Message is too - # long". Sizing on the formatted text in UTF-16 units guarantees - # every chunk is deliverable. (issue #28557) - from gateway.platforms.base import BasePlatformAdapter, utf16_len - - text_chunks = BasePlatformAdapter.truncate_message( - formatted, 4096, len_fn=utf16_len - ) - for chunk in text_chunks: + for text_ordinal, chunk in enumerate(planned_text_chunks): try: last_msg = await _send_telegram_message_with_retry( bot, @@ -1703,6 +1743,8 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No parse_mode=send_parse_mode, **text_kwargs ) elif "parse" in str(md_error).lower() or "markdown" in str(md_error).lower() or "html" in str(md_error).lower(): + if receipt_bound: + raise logger.warning( "Parse mode %s failed in _send_telegram, falling back to plain text: %s", send_parse_mode, @@ -1723,8 +1765,36 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No ) else: raise - - for media_path, is_voice in media_files: + actual_thread = text_kwargs.get("message_thread_id") + last_provider_message_id = normalize_transport_provider_message_id( + getattr(last_msg, "message_id", None) + ) + if last_provider_message_id is None: + receipts.append(TransportReceipt( + outcome="unknown", + requested_target=requested_target, + component="text", + ordinal=text_ordinal, + )) + return { + "error": "Telegram delivery acknowledgement is invalid", + "error_kind": "unknown", + "retryable": False, + "receipts": tuple(receipts), + } + receipts.append(TransportReceipt( + outcome="delivered", + provider_message_id=last_provider_message_id, + requested_target=requested_target, + actual_target=TransportTarget( + "telegram", str(chat_id), + str(actual_thread) if actual_thread is not None else None, + ), + component="text", + ordinal=text_ordinal, + )) + + for media_ordinal, (media_path, is_voice) in enumerate(media_files): if not os.path.exists(media_path): warning = f"Media file not found, skipping: {media_path}" logger.warning(warning) @@ -1732,12 +1802,34 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No # Caption mode suppressed the separate text send; if the file # it was meant to caption is gone, deliver the caption text on # its own so the words aren't silently lost. - if _tg_caption is not None and last_msg is None: + if ( + not receipt_bound + and _tg_caption is not None + and last_msg is None + ): try: last_msg = await _send_telegram_message_with_retry( bot, chat_id=int_chat_id, text=_tg_caption, parse_mode=send_parse_mode, **text_kwargs ) + last_provider_message_id = ( + normalize_transport_provider_message_id( + getattr(last_msg, "message_id", None) + ) + ) + if last_provider_message_id is None: + receipts.append(TransportReceipt( + outcome="unknown", + requested_target=requested_target, + component="text", + ordinal=0, + )) + return { + "error": "Telegram delivery acknowledgement is invalid", + "error_kind": "unknown", + "retryable": False, + "receipts": tuple(receipts), + } _tg_caption = None # delivered — don't re-caption a later file except Exception as _cap_err: logger.warning( @@ -1821,6 +1913,8 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No "parse" in str(media_err).lower() or "caption" in str(media_err).lower() ): + if receipt_bound: + raise # Caption failed to parse as MarkdownV2/HTML — # retry with a plain-text caption so the media # (and its caption) still deliver. @@ -1850,6 +1944,34 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No ) else: raise + actual_thread = media_kwargs.get("message_thread_id") + last_provider_message_id = normalize_transport_provider_message_id( + getattr(last_msg, "message_id", None) + ) + if last_provider_message_id is None: + receipts.append(TransportReceipt( + outcome="unknown", + requested_target=requested_target, + component="media", + ordinal=media_ordinal, + )) + return { + "error": "Telegram delivery acknowledgement is invalid", + "error_kind": "unknown", + "retryable": False, + "receipts": tuple(receipts), + } + receipts.append(TransportReceipt( + outcome="delivered", + provider_message_id=last_provider_message_id, + requested_target=requested_target, + actual_target=TransportTarget( + "telegram", str(chat_id), + str(actual_thread) if actual_thread is not None else None, + ), + component="media", + ordinal=media_ordinal, + )) except Exception as e: warning = _sanitize_error_text(f"Failed to send media {media_path}: {e}") logger.error(warning) @@ -1865,15 +1987,21 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No "success": True, "platform": "telegram", "chat_id": chat_id, - "message_id": str(last_msg.message_id), + "message_id": last_provider_message_id, } + if receipts: + result["receipts"] = tuple(receipts) if warnings: result["warnings"] = warnings return result except ImportError: return {"error": "python-telegram-bot not installed. Run: pip install python-telegram-bot"} except Exception as e: - return _error(f"Telegram send failed: {e}") + result = _error(f"Telegram send failed: {e}") + partial = tuple(locals().get("receipts", ())) + if partial: + result["receipts"] = partial + return result # _send_slack moved to the slack plugin as _standalone_send @@ -2254,11 +2382,16 @@ async def _send_matrix_via_adapter(pconfig, chat_id, message, media_files=None, async def _matrix_send_core(adapter, chat_id, message, media_files, metadata): """Core send logic shared by live and ephemeral Matrix adapters.""" last_result = None + receipts = [] if message.strip(): last_result = await adapter.send(chat_id, message, metadata=metadata) + receipts.extend(getattr(last_result, "receipts", ())) if not last_result.success: - return _error(f"Matrix send failed: {last_result.error}") + result = _error(f"Matrix send failed: {last_result.error}") + if receipts: + result["receipts"] = tuple(receipts) + return result for media_path, is_voice in media_files: if not os.path.exists(media_path): @@ -2276,18 +2409,25 @@ async def _matrix_send_core(adapter, chat_id, message, media_files, metadata): else: last_result = await adapter.send_document(chat_id, media_path, metadata=metadata) + receipts.extend(getattr(last_result, "receipts", ())) if not last_result.success: - return _error(f"Matrix media send failed: {last_result.error}") + result = _error(f"Matrix media send failed: {last_result.error}") + if receipts: + result["receipts"] = tuple(receipts) + return result if last_result is None: return {"error": "No deliverable text or media remained after processing MEDIA tags"} - return { + result = { "success": True, "platform": "matrix", "chat_id": chat_id, "message_id": last_result.message_id, } + if receipts: + result["receipts"] = tuple(receipts) + return result # _send_dingtalk moved to plugins/platforms/dingtalk/adapter.py::_standalone_send, diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 783d3b6bc515..a58054ad5d3d 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -607,6 +607,10 @@ export const api = { // Cron jobs getCronJobs: (profile = "all") => fetchJSON(`/api/cron/jobs?profile=${encodeURIComponent(profile)}`), + getCronJobDetail: (id: string, profile: string) => + fetchJSON( + `/api/cron/jobs/${encodeURIComponent(id)}/detail?profile=${encodeURIComponent(profile)}`, + ), getCronDeliveryTargets: () => fetchJSON<{ targets: CronDeliveryTarget[] }>("/api/cron/delivery-targets"), createCronJob: (job: CronJobMutation, profile = "default") => @@ -2265,7 +2269,7 @@ export interface CronJob { prompt?: string | null; script?: string | null; skills?: string[] | null; - schedule?: { kind?: string; expr?: string; run_at?: string; display?: string }; + schedule?: { kind?: string; expr?: string; run_at?: string | null; display?: string }; schedule_display?: string | null; repeat?: CronJobRepeat | null; enabled: boolean; @@ -2273,6 +2277,8 @@ export interface CronJob { deliver?: string | null; model?: string | null; provider?: string | null; + provider_snapshot?: string | null; + model_snapshot?: string | null; base_url?: string | null; no_agent?: boolean | null; context_from?: string[] | string | null; @@ -2283,7 +2289,15 @@ export interface CronJob { last_status?: string | null; last_error?: string | null; last_delivery_error?: string | null; - last_fire_error?: { at?: string | null; detail?: string | null } | null; + last_fire_error?: { + at?: string | null; + error_kind?: "fire_forward_failed" | null; + } | null; + delivery_kind?: "local" | "origin" | "all" | "external"; + mode?: "agent" | "script" | "monitor"; + skill_count?: number; + toolset_count?: number; + model_configured?: boolean; } export interface CronDeliveryTarget { diff --git a/web/src/lib/cron-job.ts b/web/src/lib/cron-job.ts index de4fa9fc47fb..2eb365f56a0a 100644 --- a/web/src/lib/cron-job.ts +++ b/web/src/lib/cron-job.ts @@ -1,5 +1,50 @@ import type { CronJob, CronJobMutation } from "./api"; +export function cronJobProfile(job: CronJob): string { + const profile = typeof job.profile === "string" ? job.profile : ""; + const profileName = + typeof job.profile_name === "string" ? job.profile_name : ""; + return profile || profileName || "default"; +} + +export function cronJobKey(job: CronJob): string { + return `${cronJobProfile(job)}:${job.id}`; +} + +export function splitCronJobKey(key: string): { profile: string; id: string } { + const idx = key.indexOf(":"); + if (idx === -1) return { profile: "default", id: key }; + return { profile: key.slice(0, idx) || "default", id: key.slice(idx + 1) }; +} + +export async function loadCronJobDetailForEditor( + getDetail: (id: string, profile: string) => Promise, + job: CronJob, + profile: string, +): Promise { + if (!profile || profile === "all") { + throw new Error("cron_detail_profile_required"); + } + const detail = await getDetail(job.id, profile); + return { ...detail, profile }; +} + +export function cronJobSummaryPresentation(job: CronJob): { + title: string; + mode: "agent" | "script" | "monitor"; + modelLabel: string; +} { + const name = typeof job.name === "string" ? job.name.trim() : ""; + const id = typeof job.id === "string" ? job.id : ""; + const mode = + job.mode === "script" || job.mode === "monitor" ? job.mode : "agent"; + return { + title: name || id || "Cron job", + mode, + modelLabel: job.model_configured === true ? "configured" : "", + }; +} + export interface CronJobFormState { name: string; prompt: string; diff --git a/web/src/lib/schedule.ts b/web/src/lib/schedule.ts index 22d8681d43bf..73732cc75d5f 100644 --- a/web/src/lib/schedule.ts +++ b/web/src/lib/schedule.ts @@ -323,7 +323,7 @@ export interface ScheduleLike { kind?: string; expr?: string; minutes?: number; - run_at?: string; + run_at?: string | null; display?: string; } diff --git a/web/src/pages/CronPage.test.tsx b/web/src/pages/CronPage.test.tsx new file mode 100644 index 000000000000..3aca5002d3a4 --- /dev/null +++ b/web/src/pages/CronPage.test.tsx @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + cronJobKey, + cronJobProfile, + cronJobSummaryPresentation, + loadCronJobDetailForEditor, +} from "@/lib/cron-job"; +import type { CronJob } from "@/lib/api"; + +const summary: CronJob = { + id: "job-detail-1", + name: "summary job", + enabled: true, + mode: "agent", + delivery_kind: "local", +}; + +describe("loadCronJobDetailForEditor", () => { + it("keeps identical job ids distinct across profiles", () => { + expect(cronJobKey({ ...summary, profile: "alpha" })).toBe( + "alpha:job-detail-1", + ); + expect(cronJobKey({ ...summary, profile: "beta" })).toBe( + "beta:job-detail-1", + ); + }); + + it("uses the owning profile from a summary row", () => { + expect( + cronJobProfile({ + ...summary, + profile: "alpha", + profile_name: "legacy-name", + }), + ).toBe("alpha"); + expect(cronJobProfile({ ...summary, profile_name: "legacy-name" })).toBe( + "legacy-name", + ); + }); + + it("fails closed when no concrete profile is selected", async () => { + const getDetail = vi.fn(); + + await expect( + loadCronJobDetailForEditor(getDetail, summary, "all"), + ).rejects.toThrow("cron_detail_profile_required"); + expect(getDetail).not.toHaveBeenCalled(); + }); + + it("loads editable configuration from the explicit profile detail endpoint", async () => { + const detail: CronJob = { + ...summary, + prompt: "private editable prompt", + workdir: "/private/edit-worktree", + }; + const getDetail = vi.fn().mockResolvedValue(detail); + + await expect( + loadCronJobDetailForEditor(getDetail, summary, "worker_alpha"), + ).resolves.toEqual({ ...detail, profile: "worker_alpha" }); + expect(getDetail).toHaveBeenCalledOnce(); + expect(getDetail).toHaveBeenCalledWith("job-detail-1", "worker_alpha"); + }); + + it("presents summaries without falling back to private config", () => { + const accidentalWideResponse: CronJob = { + ...summary, + name: undefined, + prompt: "PRIVATE_PROMPT", + script: "PRIVATE_SCRIPT", + provider: "PRIVATE_PROVIDER", + model: "PRIVATE_MODEL", + mode: "monitor", + model_configured: true, + }; + + expect(cronJobSummaryPresentation(accidentalWideResponse)).toEqual({ + title: "job-detail-1", + mode: "monitor", + modelLabel: "configured", + }); + }); +}); diff --git a/web/src/pages/CronPage.tsx b/web/src/pages/CronPage.tsx index a29a192dc281..3a08efc70707 100644 --- a/web/src/pages/CronPage.tsx +++ b/web/src/pages/CronPage.tsx @@ -21,8 +21,13 @@ import type { import { buildCronJobPayload, cronJobHasExecutionContent, + cronJobKey, + cronJobProfile, cronJobFormFromJob, + cronJobSummaryPresentation, + loadCronJobDetailForEditor, cronLastResult, + splitCronJobKey, type CronJobFormState, } from "@/lib/cron-job"; import { DeleteConfirmDialog } from "@/components/DeleteConfirmDialog"; @@ -68,10 +73,6 @@ function truncateText(value: string, maxLength: number): string { : value; } -function getJobPrompt(job: CronJob): string { - return asText(job.prompt); -} - function NameCheckboxPicker({ id, available, @@ -440,21 +441,8 @@ function CronJobFormFields({ ); } -function getJobName(job: CronJob): string { - return asText(job.name).trim(); -} - function getJobTitle(job: CronJob): string { - const name = getJobName(job); - if (name) return name; - - const prompt = getJobPrompt(job); - if (prompt) return truncateText(prompt, 60); - - const script = asText(job.script); - if (script) return truncateText(script, 60); - - return job.id || "Cron job"; + return cronJobSummaryPresentation(job).title; } function getJobScheduleDisplay( @@ -468,7 +456,7 @@ function getJobScheduleDisplay( // then the structured ``display`` field, then the raw ``expr``) so // legacy job rows still render *something* meaningful. return describeSchedule( - job.schedule, + job.schedule ?? undefined, asText(job.schedule_display) || asText(job.schedule?.display), strings, ); @@ -486,30 +474,11 @@ function getRepeatDisplay(job: CronJob): string { } function getJobMode(job: CronJob): string { - if (job.no_agent) return "no_agent"; - if (job.script) return "script+agent"; - return "agent"; + return cronJobSummaryPresentation(job).mode; } function getModelDisplay(job: CronJob): string { - const provider = asText(job.provider); - const model = asText(job.model); - if (provider && model) return `${provider}/${model}`; - return model || provider; -} - -function getJobProfile(job: CronJob): string { - return asText(job.profile) || asText(job.profile_name) || "default"; -} - -function getJobKey(job: CronJob): string { - return `${getJobProfile(job)}:${job.id}`; -} - -function splitJobKey(key: string): { profile: string; id: string } { - const idx = key.indexOf(":"); - if (idx === -1) return { profile: "default", id: key }; - return { profile: key.slice(0, idx) || "default", id: key.slice(idx + 1) }; + return cronJobSummaryPresentation(job).modelLabel; } function profileLabel(profile: string): string { @@ -606,12 +575,24 @@ export default function CronPage() { const [availableToolsets, setAvailableToolsets] = useState([]); const [modelOptions, setModelOptions] = useState(null); - const resourceProfile = editJob ? getJobProfile(editJob) : createProfile; + const resourceProfile = editJob ? cronJobProfile(editJob) : createProfile; - const openEditModal = useCallback((job: CronJob) => { - setEditJob(job); - setEditForm(editorFormFromJob(job)); - }, []); + const openEditModal = useCallback(async (job: CronJob) => { + const profile = cronJobProfile(job); + try { + const detail = await loadCronJobDetailForEditor( + api.getCronJobDetail, + job, + profile, + ); + setEditJob(detail); + setEditForm(editorFormFromJob(detail)); + } catch (error) { + const message = + `${t.common.loading}: ${error}`; + showToast(message, "error"); + } + }, [showToast, t.common.loading]); const selectedProfileRef = useRef(selectedProfile); const jobsRequestGenerationRef = useRef(0); @@ -742,7 +723,7 @@ export default function CronPage() { await api.updateCronJob( editJob.id, payload, - getJobProfile(editJob), + cronJobProfile(editJob), ); showToast("Saved changes ✓", "success"); setEditJob(null); @@ -757,7 +738,7 @@ export default function CronPage() { const handlePauseResume = async (job: CronJob) => { try { const isPaused = getJobState(job) === "paused"; - const profile = getJobProfile(job); + const profile = cronJobProfile(job); if (isPaused) { await api.resumeCronJob(job.id, profile); showToast( @@ -778,7 +759,7 @@ export default function CronPage() { }; const handleTrigger = async (job: CronJob) => { - const jobKey = getJobKey(job); + const jobKey = cronJobKey(job); const label = `${t.cron.triggerNow}: "${truncateText(getJobTitle(job), 30)}"`; const viewProfile = selectedProfile; const controller = triggerControllerRef.current; @@ -792,7 +773,7 @@ export default function CronPage() { // the request has not produced yet. Terminal feedback only. const result = await controller.run( jobKey, - () => api.triggerCronJob(job.id, getJobProfile(job)), + () => api.triggerCronJob(job.id, cronJobProfile(job)), ); if ( @@ -816,8 +797,8 @@ export default function CronPage() { const jobDelete = useConfirmDelete({ onDelete: useCallback( async (key: string) => { - const { profile, id } = splitJobKey(key); - const job = jobs.find((j) => getJobKey(j) === key); + const { profile, id } = splitCronJobKey(key); + const job = jobs.find((j) => cronJobKey(j) === key); try { await api.deleteCronJob(id, profile); showToast( @@ -862,7 +843,7 @@ export default function CronPage() { } const pendingJob = jobDelete.pendingId - ? jobs.find((j) => getJobKey(j) === jobDelete.pendingId) + ? jobs.find((j) => cronJobKey(j) === jobDelete.pendingId) : null; return ( @@ -1090,17 +1071,14 @@ export default function CronPage() { {jobs.map((job) => { const state = getJobState(job); - const promptText = getJobPrompt(job); const title = getJobTitle(job); - const hasName = Boolean(getJobName(job)); - const deliver = asText(job.deliver); - const profile = getJobProfile(job); - const jobKey = getJobKey(job); + const deliver = asText(job.delivery_kind); + const profile = cronJobProfile(job); + const jobKey = cronJobKey(job); const mode = getJobMode(job); const modelDisplay = getModelDisplay(job); - const toolsets = Array.isArray(job.enabled_toolsets) - ? job.enabled_toolsets.filter(Boolean) - : []; + const skillCount = job.skill_count ?? 0; + const toolsetCount = job.toolset_count ?? 0; const lastResult = cronLastResult(job); return ( @@ -1127,11 +1105,9 @@ export default function CronPage() { {deliver && deliver !== "local" && ( {deliver} )} - {Array.isArray(job.skills) && job.skills.length > 0 && ( - - {job.skills.length === 1 - ? job.skills[0] - : `${job.skills.length} skills`} + {skillCount > 0 && ( + + {skillCount === 1 ? "1 skill" : `${skillCount} skills`} )} {mode !== "agent" && ( @@ -1142,17 +1118,12 @@ export default function CronPage() { model )} - {toolsets.length > 0 && ( - - {toolsets.length} toolsets + {toolsetCount > 0 && ( + + {toolsetCount} toolsets )} - {hasName && promptText && ( -

- {truncateText(promptText, 100)} -

- )}
{getJobScheduleDisplay(job, scheduleDescribeStrings)} @@ -1167,18 +1138,17 @@ export default function CronPage() {
{job.last_delivery_error && (

- delivery: {job.last_delivery_error} + delivery: failed

)} - {job.last_fire_error?.detail && ( + {job.last_fire_error?.error_kind === "fire_forward_failed" && (

- missed scheduled fire ({formatTime(job.last_fire_error.at ?? null)}):{" "} - {job.last_fire_error.detail} + missed scheduled fire ({formatTime(job.last_fire_error.at ?? null)})

)} {job.last_error && (

- {job.last_error} + run failed

)}