From a8d4cc9f42ca90e649b2e78efde84ca97c54dc07 Mon Sep 17 00:00:00 2001 From: Anthony Ruiz Date: Wed, 26 Aug 2026 00:50:40 +0000 Subject: [PATCH 1/5] fix(webui): make replay reconciliation payload-strict --- ARCHITECTURE.md | 16 + api/models.py | 689 +++++++-- api/routes.py | 111 +- api/session_recovery.py | 55 +- api/streaming.py | 1336 ++++++++++++++--- tests/test_context_message_dedup.py | 26 +- tests/test_issue2592_partial_dedupe.py | 552 +++++++ ...issue7032_context_replay_token_identity.py | 190 +++ tests/test_issue765_streaming_persistence.py | 33 +- .../test_merge_backfill_perf_optimization.py | 244 ++- tests/test_replay_payload_strictness.py | 1324 ++++++++++++++++ tests/test_sprint42.py | 16 +- tests/test_webui_state_db_reconciliation.py | 202 ++- 13 files changed, 4336 insertions(+), 458 deletions(-) create mode 100644 tests/test_issue7032_context_replay_token_identity.py create mode 100644 tests/test_replay_payload_strictness.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ffd46b40386..01382468e5d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -260,6 +260,22 @@ usable timestamp and rows at or after the sidecar tail also append normally. The fallback therefore preserves an accepted state-only row when exact ordering is ambiguous, while safely placeable recovery rows remain chronological. +#### Replay reconciliation authority + +The visible `messages` projection, provider-facing `context_messages`, and +persisted session repair all use the same assistant replay pipeline. Adjacent +non-empty assistants collapse only when their complete strict-JSON payload +digests match; ids, timestamps, reasoning, annotations, attachments, and other +provider metadata therefore remain authoritative. Empty, partial, and +incomplete assistants use the narrower typed replay identities implemented by +that pipeline. Incomparable payloads fail closed and remain in order. + +The active-turn boundary is an atomic `(current_turn_user_idx, turn_id)` pair +owned by one completed agent attempt. A credential retry clears any pair from +the failed attempt, then accepts either a complete pair from the new result or +a complete pair from the new agent. Fields from separate attempts or sources +must never be combined into deletion authority. + #### Imported `state.db` sidebar projection `api.models.get_cli_sessions()` projects conversations from the active Hermes diff --git a/api/models.py b/api/models.py index 00f1931a4cb..eae0966d204 100644 --- a/api/models.py +++ b/api/models.py @@ -13,6 +13,7 @@ import threading import time import uuid +import weakref from contextlib import closing, contextmanager from dataclasses import dataclass from pathlib import Path @@ -253,10 +254,22 @@ def _safe_replace(src: Path, dst: Path) -> None: # Serializes index writers so concurrent Session.save() calls cannot race on # stale baselines while still allowing LOCK to be released before disk I/O. _INDEX_WRITE_LOCK = threading.RLock() +_SESSION_SAVE_AUTHORITIES_LOCK = threading.Lock() +_SESSION_SAVE_AUTHORITIES: "weakref.WeakValueDictionary[str, threading.RLock]" = weakref.WeakValueDictionary() _SESSION_INDEX_REBUILD_LOCK = threading.Lock() _SESSION_INDEX_REBUILD_THREAD = None _SESSION_INDEX_REBUILD_THREAD_TARGET: tuple[Path, Path] | None = None + +def _session_save_authority(session_id: str) -> threading.RLock: + """Return the process-wide reentrant save authority for one session ID.""" + with _SESSION_SAVE_AUTHORITIES_LOCK: + authority = _SESSION_SAVE_AUTHORITIES.get(session_id) + if authority is None: + authority = threading.RLock() + _SESSION_SAVE_AUTHORITIES[session_id] = authority + return authority + # Serializes ``_record_webui_zero_message_orphan_tombstone`` / # ``_clear_webui_zero_message_orphan_tombstone`` so two concurrent sidebar # polls (or a poll racing ``Session.save`` / ``new_session`` / @@ -506,7 +519,15 @@ def _write_session_index(updates=None, *, session_dir: Path | None = None, sessi raise ValueError("session index must be a list") with LOCK: in_memory_ids = set(SESSIONS.keys()) - updated_map = {s.session_id: s.compact() for s in updates} + # Callers may pass already-owned compact entries. In particular, + # Session.save() does so to prevent the index writer from rereading + # mutable Session state after the matching sidecar was published. + updated_map = {} + for update in updates: + entry = update if isinstance(update, dict) else update.compact() + sid = entry.get('session_id') if isinstance(entry, dict) else None + if sid: + updated_map[sid] = entry existing = [ e for e in existing @@ -1252,7 +1273,7 @@ def _load_session_from_path(path: Path) -> "Session | None": data = json.loads(path.read_text(encoding='utf-8')) except Exception: return None - data['messages'], _collapsed_partials = _collapse_adjacent_duplicate_partials(data.get('messages')) + data, _, _ = _repair_session_message_projections(data) return Session(**data) @@ -1560,6 +1581,14 @@ def path(self): return SESSION_DIR / f'{self.session_id}.json' def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: + # Distinct Session objects can represent the same durable sidecar. One + # stable SID authority therefore spans snapshot creation, sidecar + # replacement, and publication of the matching compact index row. + authority = _session_save_authority(self.session_id) + with authority: + self._save_owned_generation(touch_updated_at=touch_updated_at, skip_index=skip_index) + + def _save_owned_generation(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: if not is_safe_session_id(self.session_id): raise ValueError(f"Unsafe session_id {self.session_id!r}; refusing to write outside session store") # ── #1558 P0 guard ────────────────────────────────────────────── @@ -1579,6 +1608,13 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: f"Reload with metadata_only=False before mutating state. " f"See #1558." ) + # Persist a collapsed snapshot without rebinding or mutating the live + # list. Active workers can hold an alias to ``self.messages``; replacing + # it here would detach an append that lands while save() is preparing + # the payload. A later save will include any concurrent append. + # Own the complete snapshot BEFORE duplicate selection. Otherwise a + # nested mutation after selection but before deepcopy can invalidate the + # equality decision and leak into this save's payload (#6600). if touch_updated_at: self.updated_at = time.time() self._webui_pending_user_timestamp_identity = ( @@ -1586,6 +1622,25 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: self, getattr(self, '_webui_pending_user_timestamp_identity', None) ) ) + # Freeze every persisted/indexed field, not only messages. The sidecar + # and compact row below are projections of this one immutable generation. + generation = copy.copy(self) + generation.__dict__ = { + key: copy.deepcopy(value) + for key, value in self.__dict__.items() + } + projection_data = { + 'messages': generation.messages, + 'context_messages': generation.context_messages, + 'compression_anchor_visible_idx': generation.compression_anchor_visible_idx, + } + projection_data, messages_changed, _ = _repair_session_message_projections( + projection_data + ) + messages_to_persist = projection_data['messages'] + generation.context_messages = projection_data['context_messages'] + if messages_changed: + generation.compression_anchor_visible_idx = None # Write metadata fields first so load_metadata_only() can read them # without parsing the full messages array (which may be 400KB+). # Fields are listed in the order they should appear in the JSON file. @@ -1620,14 +1675,14 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: 'share_token', 'share_created_at', 'gateway_run', ] - meta = {k: getattr(self, k, None) for k in METADATA_FIELDS} + meta = {k: getattr(generation, k, None) for k in METADATA_FIELDS} # #5854: message_count and a compact anchor-scene fingerprint go in the # metadata prefix (BEFORE messages) so load_metadata_only() and the # sidebar-poll freshness check never have to parse the full (250-480KB) # scene bodies. message_count is placed BEFORE anchor_scene_index so a # legacy-format reader that stops at a scene key still finds the count. # The full anchor_activity_scenes bodies serialize AFTER messages. - meta['message_count'] = len(self.messages or []) + meta['message_count'] = len(messages_to_persist or []) # _mc_v marks this file as written by the current writer contract, # where `message_count` equals len(messages) by construction and both # keys land in the same atomic write. save()'s shrink guard takes the @@ -1638,20 +1693,20 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: # growth and skip the #1558 backup. One save re-marks the file, so the # fast path still covers steady state. meta['_mc_v'] = _MESSAGE_COUNT_MARKER - meta['anchor_scene_index'] = _anchor_scene_index_from_records(self.anchor_activity_scenes) + meta['anchor_scene_index'] = _anchor_scene_index_from_records(generation.anchor_activity_scenes) # Keep the in-memory fingerprint aligned with what we just persisted, so a # later metadata-only reload of THIS object (or any fingerprint reader) # sees the current value rather than a stale load-time snapshot (#5854 # defense-in-depth; the cached-side freshness check reads real records, # not this, so this is belt-and-suspenders). self._anchor_scene_index = dict(meta['anchor_scene_index']) - meta['messages'] = self.messages - meta['tool_calls'] = self.tool_calls - meta['anchor_activity_scenes'] = self.anchor_activity_scenes if isinstance(self.anchor_activity_scenes, dict) else {} + meta['messages'] = messages_to_persist + meta['tool_calls'] = generation.tool_calls + meta['anchor_activity_scenes'] = generation.anchor_activity_scenes if isinstance(generation.anchor_activity_scenes, dict) else {} # Fields not in METADATA_FIELDS (e.g. last_usage) go at the end. Exclude # the keys we placed explicitly above so they aren't emitted twice. _placed = {'message_count', '_mc_v', 'anchor_scene_index', 'messages', 'tool_calls', 'anchor_activity_scenes'} - extra = {k: v for k, v in self.__dict__.items() + extra = {k: v for k, v in generation.__dict__.items() if k not in METADATA_FIELDS and k not in _placed and not k.startswith('_')} payload = json.dumps({**meta, **extra}, ensure_ascii=False, indent=2) @@ -1710,11 +1765,11 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: existing_msg_count = len(existing.get('messages') or []) except (json.JSONDecodeError, ValueError): existing_msg_count = -1 # corrupt → always back up - incoming_msg_count = len(self.messages or []) + incoming_msg_count = len(messages_to_persist or []) if ( existing_msg_count > 0 and incoming_msg_count == 0 - and (self.active_stream_id or self.pending_user_message) + and (generation.active_stream_id or generation.pending_user_message) ): logger.warning( "refusing to overwrite session %s messages with empty active/pending snapshot " @@ -1722,7 +1777,7 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: self.session_id, existing_msg_count, incoming_msg_count, - self.active_stream_id, + generation.active_stream_id, ) return if existing_msg_count > incoming_msg_count: @@ -1771,7 +1826,12 @@ def save(self, touch_updated_at: bool = True, skip_index: bool = False) -> None: pass raise if not skip_index: - _write_session_index(updates=[self]) + # #6600: project the sidebar index from the SAME detached snapshot + # just serialized — never from the live list — so _index.json can + # neither record the uncollapsed message_count nor adopt a dropped + # duplicate row's later timestamp. + index_entry = generation.compact(projection_messages=messages_to_persist) + _write_session_index(updates=[index_entry]) # #4985 belt-and-suspenders self-heal: a successful save with at # least one real message on the sidecar is unconditional proof the @@ -1812,16 +1872,16 @@ def load(cls, sid): # during the parse (TOCTOU guard against an atomic replace mid-read). _pre_read_sig = _sidecar_stat_signature(p) data = json.loads(p.read_text(encoding='utf-8')) - data['messages'], _collapsed_partials = _collapse_adjacent_duplicate_partials(data.get('messages')) + data, messages_changed, context_changed = _repair_session_message_projections(data) session = cls(**data) - if _collapsed_partials: + if messages_changed or context_changed: try: # Self-heal bloated sessions on first full load without touching # recency/index ordering; save() creates a .bak because this # intentionally shrinks the transcript (#2592). session.save(touch_updated_at=False, skip_index=True) except Exception: - logger.debug("Failed to persist collapsed duplicate partials for %s", sid, exc_info=True) + logger.debug("Failed to persist collapsed duplicate assistant rows for %s", sid, exc_info=True) else: # #5854: for a LEGACY sidecar (no modern anchor_scene_index key), the # cheap metadata-prefix read cannot recover message_count/scenes when @@ -1831,8 +1891,8 @@ def load(cls, sid): # Keyed by stat signature, so any edit invalidates it; the next # save() rewrites the modern layout and the fallback stops firing. # expected_sig guards against an atomic replace during the read. - # (When _collapsed_partials fired, save() above already rewrote the - # modern layout, so no legacy caching is needed.) + # (When either duplicate-row repair fired, save() above already + # rewrote the modern layout, so no legacy caching is needed.) # # WHY STILL GATED, and not "obviously" broadened to modern files: the # cache is only READ from the two sites that reach it when the cheap @@ -1987,18 +2047,29 @@ def compact( include_runtime=False, active_stream_ids=None, sidebar_metadata_only=False, + projection_messages=None, ) -> dict: active_stream_ids = active_stream_ids if active_stream_ids is not None else set() has_pending_user_message = bool(self.pending_user_message) - message_count = ( - self._metadata_message_count - if self._metadata_message_count is not None - else len(self.messages) - ) - if has_pending_user_message: + # #6600: during save()'s index update this is the detached, collapsed + # snapshot just written to the sidecar, keeping the index projection + # identical to the persisted payload. Outside save() it is None and + # the live list is used as before. + has_index_projection = projection_messages is not None + if not has_index_projection: + projection_messages = self.messages + if has_index_projection: + message_count = len(projection_messages) + else: + message_count = ( + self._metadata_message_count + if self._metadata_message_count is not None + else len(projection_messages) + ) + if has_pending_user_message and not has_index_projection: message_count = max(message_count, 1) - last_message_at = _last_message_timestamp(self.messages) or self.updated_at - if has_pending_user_message and self.pending_started_at: + last_message_at = _last_message_timestamp(projection_messages) or self.updated_at + if has_pending_user_message and self.pending_started_at and not has_index_projection: last_message_at = self.pending_started_at compact = { 'session_id': self.session_id, @@ -2058,7 +2129,7 @@ def compact( 'worktree_repo_root': self.worktree_repo_root, 'worktree_created_at': self.worktree_created_at, } if self.worktree_path else {}), - 'user_message_count': Session._compute_user_message_count(self.messages), + 'user_message_count': Session._compute_user_message_count(projection_messages), 'active_stream_id': self.active_stream_id, 'pending_user_message': self.pending_user_message, 'has_pending_user_message': has_pending_user_message, @@ -2800,35 +2871,16 @@ def _transcript_already_advanced_past_pending(session) -> bool: return False -def _partial_message_signature(message: dict) -> tuple: - """Return a stable identity for partial assistant markers recovered on load.""" +def _partial_message_signature(message: dict) -> bytes | None: + """Return an exact JSON identity for a partial marker, or ``None``. + + Partial rows can carry durable ids, attachments, provider sidecars, and + structured tool state. Reducing on a hand-picked subset can erase a + distinct row before the stricter reducers ever see it. + """ if not isinstance(message, dict): - return ('', '', ()) - tool_sig = [] - for tool_call in message.get('_partial_tool_calls') or []: - if not isinstance(tool_call, dict): - continue - try: - args_sig = json.dumps( - tool_call.get('args') or {}, - ensure_ascii=False, - sort_keys=True, - default=str, - ) - except Exception: - args_sig = str(tool_call.get('args') or '') - tool_sig.append(( - str(tool_call.get('name') or ''), - args_sig, - bool(tool_call.get('done', False)), - bool(tool_call.get('is_error', False)), - str(tool_call.get('preview') or tool_call.get('snippet') or ''), - )) - return ( - str(message.get('content') or '').strip(), - str(message.get('reasoning') or '').strip(), - tuple(tool_sig), - ) + return None + return _canonical_message_digest(message) def _collapse_adjacent_duplicate_partials(messages) -> tuple[list, bool]: @@ -2841,7 +2893,7 @@ def _collapse_adjacent_duplicate_partials(messages) -> tuple[list, bool]: for message in messages: if isinstance(message, dict) and message.get('_partial'): sig = _partial_message_signature(message) - if previous_partial_sig == sig: + if sig is not None and previous_partial_sig == sig: changed = True continue previous_partial_sig = sig @@ -2851,6 +2903,351 @@ def _collapse_adjacent_duplicate_partials(messages) -> tuple[list, bool]: return collapsed, changed +def _strict_incomplete_message_id_key(message_id): + """Return a type-tagged deletion key for a persisted message id, or None. + + Only exact ``str``/``int``/finite-``float`` scalars carry deletion + authority. Booleans, containers, subclass instances (e.g. enum-like + ids), and non-finite floats are rejected so distinct typed ids such as + ``1`` vs ``"1"`` or ``True`` vs ``"True"`` can never collapse into the + same bucket — a genuinely distinct backup row must never be classified + as a duplicate-only replay (#6600 review). + """ + if isinstance(message_id, bool): + return None + if type(message_id) is str: + return ('str', message_id) if message_id != '' else None + if type(message_id) is int: + return ('int', message_id) + if type(message_id) is float: + return ('float', message_id) if math.isfinite(message_id) else None + return None + + +def _strip_thinking_markup_for_incomplete(text: str) -> str: + """Emptiness-equivalent mirror of api.streaming._strip_thinking_markup. + + Keep the regexes in sync with api.streaming._strip_thinking_markup; the + durable incomplete-row predicate must consider exactly the same content + "blank" as the reconciliation layer. Duplicated (rather than imported) + because api.streaming already imports api.models at module load, and the + .bak recovery path must not depend on the streaming import chain. + Parity is pinned by tests/test_issue2592_partial_dedupe.py. + """ + if not text: + return '' + s = str(text) + s = re.sub(r'^\s*.*?\s*', ' ', s, flags=re.IGNORECASE | re.DOTALL) + s = re.sub(r'^\s*<\|channel\|?>thought\n?.*?\s*', ' ', s, flags=re.IGNORECASE | re.DOTALL) + s = re.sub(r'^\s*<\|turn\|>thinking\n.*?\s*', ' ', s, flags=re.IGNORECASE | re.DOTALL) # Gemma 4 + s = re.sub(r'^\s*(the|ther)\s+user\s+is\s+asking[^\n]*(?:\n|$)', ' ', s, flags=re.IGNORECASE) + s = re.sub( + r"^\s*(?:here(?:'s| is) (?:a |my )?(?:thinking|thought) (?:process|trace|through)\b[^\n]*\n?" + r"|let me (?:think|work|reason|analyze|walk) (?:through|about|this|step)\b[^\n]*\n?" + r"|i(?:'ll| will) (?:think|work|reason|analyze|break this down)\b[^\n]*\n?" + r"|(?:okay|alright|sure|of course),?\s+let me\b[^\n]*\n?)", + ' ', s, flags=re.IGNORECASE + ) + s = re.sub(r'\s+', ' ', s).strip() + return s + + +def _incomplete_message_content_text(content) -> str: + """Extract visible text for the incomplete-row eligibility predicate. + + Mirrors api.streaming._message_text (structured content parts + + thinking-markup stripping) so the persistence boundary empties exactly + the rows the reconciliation layer empties (#6600 review: one shared + eligibility semantics for both layers). + """ + if isinstance(content, list): + parts = [] + for part in content: + if not isinstance(part, dict): + continue + part_type = str(part.get('type') or '').lower() + if part_type in ('', 'text', 'input_text', 'output_text'): + parts.append(str( + part.get('text') or part.get('content') or part.get('input_text') or part.get('output_text') or '' + )) + return _strip_thinking_markup_for_incomplete('\n'.join(parts).strip()) + return _strip_thinking_markup_for_incomplete(str(content or '').strip()) + + +def _is_admissible_empty_text_content(content) -> bool: + """Return True only for empty content shapes safe for replay removal.""" + if content is None: + return True + if type(content) is str: + return not bool(_incomplete_message_content_text(content)) + if type(content) is not list or not content: + return False + value_fields = { + 'text': ('text', 'content'), + 'input_text': ('input_text', 'text', 'content'), + 'output_text': ('output_text', 'text', 'content'), + } + allowed_keys = { + 'type', 'text', 'content', 'input_text', 'output_text', 'annotations', + } + for part in content: + if type(part) is not dict or any(type(key) is not str for key in part): + return False + part_type = part.get('type') + if type(part_type) is not str: + return False + part_type = part_type.lower() + if part_type not in value_fields or not set(part).issubset(allowed_keys): + return False + present_values = [ + part[field] + for field in value_fields[part_type] + if field in part + ] + if len(present_values) != 1 or type(present_values[0]) is not str: + return False + annotations = part.get('annotations') + if annotations not in (None, [], {}): + return False + return not bool(_incomplete_message_content_text(content)) + + +def _incomplete_reasoning_message_id(message): + """Return typed id plus exact payload digest for an empty incomplete result. + + This is the SINGLE eligibility predicate shared by the persistence + boundary (Session.save/load and .bak recovery) and the in-memory + reconciliation layer (api.streaming._message_identity): a row one layer + collapses is exactly a row the other layer collapses, so neither can + drop a row the other considers distinct. Returns a hashable type-tagged + tuple with a type-faithful digest, or None when the row is not eligible. + """ + if not isinstance(message, dict) or message.get('role') != 'assistant': + return None + if str(message.get('finish_reason') or '').lower() != 'incomplete': + return None + if not _is_admissible_empty_text_content(message.get('content')): + return None + if _message_has_structured_replay_fields(message): + return None + typed_id_key = _strict_incomplete_message_id_key(message.get('id')) + if typed_id_key is None: + return None + payload_digest = _canonical_message_digest(message) + if payload_digest is None: + return None + return ('message_id', typed_id_key, payload_digest) + + +def _message_information_score(message) -> int: + """Prefer the richest replay when one stable incomplete id occurs repeatedly.""" + if not isinstance(message, dict): + return 0 + try: + return len(json.dumps(message, sort_keys=True, ensure_ascii=False, default=str)) + except (TypeError, ValueError): + return len(str(message)) + + +def _collapse_duplicate_incomplete_message_ids(messages) -> tuple[list, bool]: + """Collapse non-adjacent replays of empty incomplete assistant message ids.""" + if not isinstance(messages, list): + return messages, False + collapsed = [] + seen_indexes = {} + changed = False + for message in messages: + message_id = _incomplete_reasoning_message_id(message) + if message_id is None: + collapsed.append(message) + continue + existing_index = seen_indexes.get(message_id) + if existing_index is None: + seen_indexes[message_id] = len(collapsed) + collapsed.append(message) + continue + changed = True + existing = collapsed[existing_index] + if message == existing: + continue + if _message_information_score(message) > _message_information_score(existing): + collapsed[existing_index] = message + return collapsed, changed + + +def _is_strict_json_tree(value) -> bool: + """Return whether ``value`` preserves its Python identity through JSON.""" + if value is None or type(value) in (str, int, bool): + return True + if type(value) is float: + return math.isfinite(value) + if type(value) is list: + return all(_is_strict_json_tree(item) for item in value) + if type(value) is dict: + return all( + type(key) is str and _is_strict_json_tree(item) + for key, item in value.items() + ) + return False + + +def _canonical_message_digest(message): + """Hash one strictly JSON-preserving message, or fail closed.""" + if not _is_strict_json_tree(message): + return None + try: + canonical_payload = json.dumps( + message, + ensure_ascii=False, + sort_keys=True, + separators=(',', ':'), + allow_nan=False, + ).encode('utf-8') + except (TypeError, ValueError): + return None + return hashlib.sha256(canonical_payload).digest() + + +def _collapse_adjacent_exact_assistant_replays(messages) -> tuple[list, bool]: + """Collapse only adjacent assistants with identical strict JSON payloads.""" + if not isinstance(messages, list): + return messages, False + collapsed = [] + changed = False + for message in messages: + message_digest = ( + _canonical_message_digest(message) + if type(message) is dict and message.get('role') == 'assistant' + else None + ) + previous_digest = ( + _canonical_message_digest(collapsed[-1]) + if collapsed + and type(collapsed[-1]) is dict + and collapsed[-1].get('role') == 'assistant' + else None + ) + if message_digest is not None and message_digest == previous_digest: + changed = True + continue + collapsed.append(message) + return collapsed, changed + + +_STRUCTURED_REPLAY_FIELDS = frozenset({ + 'tool_call_id', + 'tool_calls', + 'function_call', + 'function_calls', + '_partial_tool_calls', + 'refusal', + 'attachments', +}) + + +def _message_has_structured_replay_fields(message): + """Return True for empty assistants whose structure is non-comparable.""" + return bool( + isinstance(message, dict) + and message.get('role') == 'assistant' + and _is_admissible_empty_text_content(message.get('content')) + and any(field in message for field in _STRUCTURED_REPLAY_FIELDS) + ) + + +def _durable_empty_assistant_replay_key(message): + """Return strict identity for a non-incomplete empty assistant replay.""" + if not isinstance(message, dict) or message.get('role') != 'assistant': + return None + if str(message.get('finish_reason') or '').lower() == 'incomplete': + return None + if message.get('_partial'): + return None + if not _is_admissible_empty_text_content(message.get('content')): + return None + if _message_has_structured_replay_fields(message): + return None + payload_digest = _canonical_message_digest(message) + if payload_digest is None: + return None + typed_id_key = _strict_incomplete_message_id_key(message.get('id')) + if typed_id_key is not None: + return ('message_id', typed_id_key, payload_digest) + recovered_stream_key = _strict_incomplete_message_id_key( + message.get('_recovered_stream_id') + ) + timestamp = ( + message.get('timestamp') + if message.get('timestamp') is not None + else message.get('_ts') + ) + recovered_timestamp_key = _strict_incomplete_message_id_key(timestamp) + if recovered_stream_key is not None and recovered_timestamp_key is not None: + return ( + 'recovered', + recovered_stream_key, + recovered_timestamp_key, + payload_digest, + ) + return None + + +def _collapse_duplicate_durable_empty_assistant_replays(messages) -> tuple[list, bool]: + """Collapse replayed empty assistant rows with an exact durable identity.""" + if not isinstance(messages, list): + return messages, False + collapsed = [] + seen_indexes = {} + changed = False + for message in messages: + replay_key = _durable_empty_assistant_replay_key(message) + if replay_key is None: + collapsed.append(message) + continue + existing_index = seen_indexes.get(replay_key) + if existing_index is None: + seen_indexes[replay_key] = len(collapsed) + collapsed.append(message) + continue + changed = True + existing = collapsed[existing_index] + if message == existing: + continue + if _message_information_score(message) > _message_information_score(existing): + collapsed[existing_index] = message + return collapsed, changed + + +def _collapse_replayed_assistant_rows(messages) -> tuple[list, bool]: + """Apply the complete replay-repair contract to one message projection.""" + repaired, exact_changed = _collapse_adjacent_exact_assistant_replays(messages) + repaired, partials_changed = _collapse_adjacent_duplicate_partials(repaired) + repaired, incomplete_changed = _collapse_duplicate_incomplete_message_ids(repaired) + repaired, durable_changed = _collapse_duplicate_durable_empty_assistant_replays( + repaired + ) + return repaired, bool( + exact_changed or partials_changed or incomplete_changed or durable_changed + ) + + +def _repair_session_message_projections(data: dict) -> tuple[dict, bool, bool]: + """Repair visible and model-facing projections with the same pipeline.""" + if not isinstance(data, dict): + return data, False, False + data['messages'], messages_changed = _collapse_replayed_assistant_rows( + data.get('messages') + ) + context_changed = False + if isinstance(data.get('context_messages'), list) and data['context_messages']: + data['context_messages'], context_changed = _collapse_replayed_assistant_rows( + data['context_messages'] + ) + if messages_changed: + data['compression_anchor_visible_idx'] = None + return data, messages_changed, context_changed + + def _find_existing_assistant_for_journal_content( session, content: str, @@ -10803,31 +11200,35 @@ def _merge_session_display_metadata(target: dict | None, source: dict | None) -> target[key] = copy.deepcopy(value) -def _state_db_row_identity_details(message: dict | None) -> tuple[str | None, bool]: - """Return ``(row_id, valid)`` for private state.db provenance aliases. +def _state_db_row_identity_details( + message: dict | None, +) -> tuple[tuple[str, object] | None, bool]: + """Return a strict typed identity for private state.db provenance aliases. - A message carrying two different aliases is contradictory provenance. It - must not silently fall through to timestamp/sequence matching, because that - would turn an identity conflict into a guessed provider-side payload. + A message carrying aliases with different scalar types or values is + contradictory provenance. Numeric spellings are normalized only within + their type bucket, so ``1``, ``1.0`` and ``"1"`` remain distinct durable + identities. Malformed provenance must not authorize deletion through a + weaker timestamp/sequence fallback. """ if not isinstance(message, dict): return None, True - values = set() + values: set[tuple[str, object]] = set() for key in ("_row_id", "_state_db_row_id", "_db_row_id", "state_db_row_id"): if key not in message or message.get(key) in (None, ""): continue value = message.get(key) - if isinstance(value, bool): - return None, False - if isinstance(value, int): - normalized = str(value) if value >= 0 else None - elif isinstance(value, float): - normalized = str(int(value)) if math.isfinite(value) and value >= 0 and value.is_integer() else None - elif isinstance(value, str): + if type(value) is int: + normalized = ("int", value) if value >= 0 else None + elif type(value) is float: + normalized = ( + ("float", value) + if math.isfinite(value) and value >= 0 and value.is_integer() + else None + ) + elif type(value) is str: text = value.strip() - normalized = text if text.isdigit() else None - if normalized is not None: - normalized = str(int(normalized)) + normalized = ("str", str(int(text))) if text.isdigit() else None else: normalized = None if normalized is None: @@ -11419,32 +11820,42 @@ def _reconcile_api_content_sidecars(sidecar_messages: list, state_messages: list def _session_message_dedup_key(msg: dict): """Like _session_message_merge_key but preserves full-precision timestamp. - Two messages are true duplicates only if role, content, AND exact - timestamp all match. Sub-second timestamp differences indicate - legitimately distinct messages (e.g. two assistant turns within the - same wall-clock second). + Two messages are true duplicates only if role, content, exact timestamp, + and any durable State DB row identity all match. Sub-second timestamp + differences indicate legitimately distinct messages (e.g. two assistant + turns within the same wall-clock second). """ if not isinstance(msg, dict): return ("non_dict", repr(msg)) + row_id, row_id_valid = _state_db_row_identity_details(msg) message_identity = msg.get("id") or msg.get("message_id") if message_identity: - return _session_message_key_with_sidecar( + key = _session_message_key_with_sidecar( ("message_id", str(message_identity)), msg ) - # Include tool_calls in the key so assistant messages that carry - # different tool invocations (but identical empty content/timestamp) - # are never collapsed into one. (#3346 regression) - _tc = msg.get("tool_calls") - _tc_key = json.dumps(_tc, sort_keys=True, default=str) if _tc else "" - return _session_message_key_with_sidecar(( - "legacy", - str(msg.get("role") or ""), - _content_identity_for_key(msg.get("content")), - str(msg.get("timestamp") or ""), - str(msg.get("tool_call_id") or ""), - str(msg.get("tool_name") or msg.get("name") or ""), - _tc_key, - ), msg) + else: + # Include tool_calls in the key so assistant messages that carry + # different tool invocations (but identical empty content/timestamp) + # are never collapsed into one. (#3346 regression) + _tc = msg.get("tool_calls") + _tc_key = json.dumps(_tc, sort_keys=True, default=str) if _tc else "" + key = _session_message_key_with_sidecar(( + "legacy", + str(msg.get("role") or ""), + _content_identity_for_key(msg.get("content")), + str(msg.get("timestamp") or ""), + str(msg.get("tool_call_id") or ""), + str(msg.get("tool_name") or msg.get("name") or ""), + _tc_key, + ), msg) + if not row_id_valid: + # Malformed provenance cannot delete another source row. Object identity + # is process-local by design and only scopes this fail-closed key to the + # current linear merge pass. + return (*key, ("invalid_state_db_row_id", id(msg))) + if row_id is not None: + return (*key, ("state_db_row_id", row_id)) + return key def _normalized_session_message_content(msg: dict): @@ -12589,15 +13000,30 @@ def _state_row_is_truncated( state_replay_idx += 1 seen_dedup_keys.add(dedup_key) continue + row_id, row_id_valid = _state_db_row_identity_details(msg) replays_sidecar_prefix = False replay_target = None if state_replay_idx < len(sidecar_visible_sequence): expected_visible_key = sidecar_visible_sequence[state_replay_idx] - if visible_key == expected_visible_key or _has_visible_duplicate( - visible_key, {expected_visible_key} + expected_message = sidecar_visible_messages[state_replay_idx] + expected_row_id, expected_row_id_valid = _state_db_row_identity_details( + expected_message + ) + row_ids_can_replay = ( + row_id_valid + and expected_row_id_valid + and ( + row_id is None + or expected_row_id is None + or row_id == expected_row_id + ) + ) + if row_ids_can_replay and ( + visible_key == expected_visible_key + or _has_visible_duplicate(visible_key, {expected_visible_key}) ): replays_sidecar_prefix = True - replay_target = sidecar_visible_messages[state_replay_idx] + replay_target = expected_message state_replay_idx += 1 if replays_sidecar_prefix: _merge_session_display_metadata(replay_target, msg) @@ -12614,7 +13040,6 @@ def _state_row_is_truncated( # are caught by the dedup guard (#3346). seen_dedup_keys.add(dedup_key) continue - row_id, row_id_valid = _state_db_row_identity_details(msg) row_id_sidecar_conflict = False existing = ( merged_by_row_id.get(row_id) @@ -12640,6 +13065,53 @@ def _state_row_is_truncated( _copy_api_content_sidecar(existing, msg) _merge_session_display_metadata(existing, msg) continue + # A unique typed state.db identity proves this is another source row + # once the sidecar's corresponding visible-occurrence budget has been + # consumed (or when the sidecar already carries different typed row + # identities). We only use that proof to bypass weaker duplicate + # heuristics below; edit/truncation watermark guards still apply. + row_id_preserves_source_multiplicity = ( + row_id_valid + and row_id is not None + and state_row_id_counts.get(row_id, 0) == 1 + and ( + ( + bool(sidecar_row_id_counts) + and sidecar_row_id_counts.get(row_id, 0) == 0 + ) + or ( + sidecar_visible_counts.get(visible_key, 0) > 0 + and skipped_state_visible_counts.get(visible_key, 0) + >= sidecar_visible_counts[visible_key] + ) + ) + ) + # Skip rows ABOVE the watermark only while the sidecar has NOT advanced + # past the watermark. Because Session.save() no longer auto-clears the + # watermark, an unconditional `timestamp > watermark` skip would become + # permanent and silently drop legitimate future state.db-only recovery + # rows once the session moves forward past the edit boundary. Once the + # sidecar's own max timestamp is beyond the watermark (the session has + # advanced), allow state rows newer than the sidecar tail to merge. + # + # The sidecar's max timestamp can also EQUAL the watermark when the new + # post-edit USER turn has been checkpointed into the sidecar (its + # timestamp == the advanced watermark) but its ASSISTANT reply exists + # only in state.db (recovery before the sidecar tail advances). In that + # state truncation_boundary < watermark proves the session is genuinely + # advanced, so the post-watermark state-only reply is legitimate + # post-edit content and must merge through (not be dropped as a replaced + # tail). The conservative skip still applies for boundary is None and + # boundary == watermark (not-advanced / legacy). + # + # CRITICAL: the boundary-advanced signal may only bypass the skip AFTER + # state replay has consumed the sidecar's visible checkpoint + # (state_replay_idx >= len(sidecar_visible_sequence)). A deleted suffix + # row with ts > watermark that appears in state.db BEFORE the edited + # checkpoint must still be skipped — otherwise the advanced signal would + # resurrect it. The sidecar-max-timestamp signal needs no such gate (a + # sidecar tail beyond the watermark is itself proof the checkpoint has + # advanced). checkpoint_consumed = state_replay_idx >= len(sidecar_visible_sequence) if _state_row_is_truncated( msg, key, content_key, timestamp, checkpoint_consumed, @@ -12658,7 +13130,11 @@ def _state_row_is_truncated( # already seen. For legacy keys the dedup check above already # handled true duplicates; same-second distinct messages must # fall through. - if key in seen_message_keys and key[0] == "message_id": + if ( + key in seen_message_keys + and key[0] == "message_id" + and not row_id_preserves_source_multiplicity + ): _merge_session_display_metadata(merged_by_message_key.get(key), msg) continue if not (isinstance(key, tuple) and key[:1] == ("message_id",)): @@ -12667,10 +13143,19 @@ def _state_row_is_truncated( # Different tool_calls produce different merge_keys even with # identical content/timestamp, so an unchecked continue here # would drop legitimately distinct turns. (#3346 / PR #3665) - if key in seen_message_keys: - _merge_session_display_metadata(merged_by_message_key.get(key), msg) + if ( + key in seen_message_keys + and not row_id_preserves_source_multiplicity + ): + _merge_session_display_metadata( + merged_by_message_key.get(key), msg + ) continue - if key in seen_message_keys and key[0] == "message_id": + if ( + key in seen_message_keys + and key[0] == "message_id" + and not row_id_preserves_source_multiplicity + ): _merge_session_display_metadata(merged_by_message_key.get(key), msg) continue matched_visible_key = _matching_visible_duplicate( @@ -12681,7 +13166,10 @@ def _state_row_is_truncated( if matched_visible_key is not None: skipped_count = skipped_state_visible_counts.get(matched_visible_key, 0) sidecar_count = sidecar_visible_counts.get(matched_visible_key, 0) - if skipped_count < sidecar_count: + if ( + skipped_count < sidecar_count + and not row_id_preserves_source_multiplicity + ): skipped_state_visible_counts[matched_visible_key] = skipped_count + 1 _merge_session_display_metadata(merged_by_visible_key.get(matched_visible_key), msg) continue @@ -12701,6 +13189,7 @@ def _state_row_is_truncated( and timestamp is not None and timestamp <= max_sidecar_timestamp and not row_id_sidecar_conflict + and not row_id_preserves_source_multiplicity ): # When a truncation watermark is active and the sidecar holds only # the edited user checkpoint, state.db may contain an assistant/tool diff --git a/api/routes.py b/api/routes.py index b5f87fc7252..444f61fdcf2 100644 --- a/api/routes.py +++ b/api/routes.py @@ -25418,18 +25418,11 @@ def _handle_chat_sync(handler, body): ) from api.streaming import ( _WEBUI_PROGRESS_PROMPT, - _active_turn_boundary, - _assign_stable_message_ids, - _dedupe_replayed_context_messages, - _find_active_turn_checkpoint_index, - _merge_display_messages_after_agent_result, _resolve_active_turn_authority, - _restore_display_reasoning_metadata, - _restore_reasoning_metadata_before_boundary, - _settle_current_turn_boundary, _sanitize_messages_for_agent, _compact_session_image_parts_for_persistence, _context_messages_for_new_turn, + _settle_result_messages, _workspace_context_prefix, ) workspace_ctx = _workspace_context_prefix(str(s.workspace)) @@ -25454,6 +25447,22 @@ def _handle_chat_sync(handler, body): _previous_messages = list(s.messages or []) _previous_context_messages = list(_context_messages_for_new_turn(s, msg)) + _sync_turn_source = getattr(s, "pending_user_source", None) or "webui" + # Synchronous requests have no SSE stream token, but they still + # need an explicit request-local turn identity. The Agent's + # persisted user index + turn id complete this provenance after + # run_conversation returns; strict settlement then uses the same + # authority as the asynchronous path instead of visible-text + # prefix inference. + _sync_active_turn_identity = { + "token": f"sync:{uuid.uuid4().hex}", + "text": msg, + "timestamp": time.time(), + "source": _sync_turn_source, + "attachments": [], + "current_turn_user_idx": None, + "turn_id": "", + } result = agent.run_conversation( user_message=workspace_ctx + msg, @@ -25484,79 +25493,33 @@ def _handle_chat_sync(handler, body): os.environ["HERMES_SESSION_KEY"] = old_session_key with _get_session_agent_lock(s.session_id): _result_messages = result.get("messages") or _previous_context_messages - # Active-turn boundary is fixed BEFORE any restoration (same as streaming), - # using whatever exact turn authority the result/Agent pair exported. - _active_turn_identity = _resolve_active_turn_authority( - {"token": None, "text": msg, "current_turn_user_idx": None, "turn_id": ""}, + _sync_active_turn_identity = _resolve_active_turn_authority( + _sync_active_turn_identity, result=result, agent=agent, ) - if ( - isinstance(_active_turn_identity, dict) - and _active_turn_identity.get("agent_turn_boundary_resolved") is True - and not _active_turn_identity.get("token") - ): - _active_image_index = _find_active_turn_checkpoint_index( - _result_messages, - _previous_context_messages, - _active_turn_identity, - msg, - ) - _active_image_content = ( - _result_messages[_active_image_index].get("content") - if _active_image_index is not None - else None - ) - if isinstance(_active_image_content, list) and any( - isinstance(part, dict) - and part.get("type") in {"image", "image_url", "input_image"} - for part in _active_image_content - ): - from api.process_event_utils import build_active_turn_token - - _active_turn_identity["token"] = build_active_turn_token( - f"sync:{s.session_id}:{_active_turn_identity['turn_id']}", - time.time(), - ) - _turn_boundary = _active_turn_boundary( - _result_messages, _previous_context_messages, _active_turn_identity, msg, - ) - _next_context_messages = _restore_reasoning_metadata_before_boundary( - _previous_context_messages, - _result_messages, - _turn_boundary, - ) - # Mint ids on the shared result rows BEFORE dedupe deep-copies any - # stale-user boundary row, so both arrays share the id (#5564). - _assign_stable_message_ids( - _result_messages, _previous_messages, _previous_context_messages - ) - _next_context_messages = _dedupe_replayed_context_messages( - _previous_context_messages, - _next_context_messages, - msg, - ) - if _active_turn_identity.get("token"): - _next_context_messages = _settle_current_turn_boundary( - _previous_context_messages, - _next_context_messages, - _active_turn_identity, - msg, - getattr(s, "pending_user_source", None) or "webui", - ) - s.context_messages = _next_context_messages - s.messages = _merge_display_messages_after_agent_result( + _settle_result_messages( + s, _previous_messages, _previous_context_messages, - _restore_display_reasoning_metadata( - _previous_messages, _result_messages, current_turn_boundary=_turn_boundary, - ), + _result_messages, msg, - source=getattr(s, "pending_user_source", None) or "webui", - verification_nudge_provenance={ - "active_turn_identity": _active_turn_identity, - }, + _sync_turn_source, + _sync_active_turn_identity, ) + # The synchronous endpoint has no reconnectable stream. Its request- + # local token is useful only while the shared settlement pipeline aligns + # display/context ownership; do not persist it as durable transcript + # metadata after the request has reached a terminal result. + _sync_turn_token = _sync_active_turn_identity.get("token") + if _sync_turn_token: + for _projection in (s.messages, s.context_messages): + for _message in _projection or []: + if ( + isinstance(_message, dict) + and _message.get("_active_turn_token") == _sync_turn_token + ): + _message.pop("_active_turn_token", None) _compact_session_image_parts_for_persistence(s) # Only auto-generate title when still default; preserves user renames if s.title == "Untitled": diff --git a/api/session_recovery.py b/api/session_recovery.py index d9b09479977..60241008f25 100644 --- a/api/session_recovery.py +++ b/api/session_recovery.py @@ -30,7 +30,6 @@ import logging import os import re -import shutil import sqlite3 import threading from contextlib import closing @@ -65,7 +64,7 @@ def _is_valid_intentional_shrink_generation(value) -> bool: def _msg_count(p: Path) -> int: - """Return the number of messages in a session JSON file, or -1 on read/parse error. + """Return the effective message count, or -1 on read/parse error. Returns -1 for any non-session-shape file: - File can't be read (OSError) @@ -82,7 +81,21 @@ def _msg_count(p: Path) -> int: if not isinstance(data, dict): return -1 msgs = data.get('messages') - return len(msgs) if isinstance(msgs, list) else -1 + if not isinstance(msgs, list): + return -1 + # A shrink caused only by collapsing replayed empty ``incomplete`` rows is + # an intentional repair, not data loss. Compare live and backup using the + # same narrow identity rule as Session.save() so startup recovery does not + # resurrect the amplification. Unique backup messages still increase the + # effective count and remain recoverable. + try: + from api.models import _collapse_replayed_assistant_rows + + msgs, _ = _collapse_replayed_assistant_rows(msgs) + except Exception: + logger.debug("Failed to compute effective recovery message count for %s", p, exc_info=True) + return -1 + return len(msgs) def _rebuild_recovery_session_index(session_dir: Path) -> None: @@ -408,13 +421,28 @@ def recover_session(session_path: Path) -> dict: if status["recommend"] != "restore": return {**status, "restored": False} bak_path = session_path.with_suffix('.json.bak') - # Stage the recovery via a tmp copy + atomic replace so a crash mid-restore - # cannot leave a half-written session.json. + # Stage the recovery via a tmp write + atomic replace so a crash + # mid-restore cannot leave a half-written session.json. tmp_path = session_path.with_suffix('.json.recover.tmp') try: - shutil.copyfile(bak_path, tmp_path) + # #6600: restore the SAME effective payload that _msg_count() + # evaluated — collapse replayed empty ``incomplete`` rows and + # recompute message_count from the collapsed list — so recovery never + # resurrects the duplicate amplification it just decided to repair. + bak_data = json.loads(bak_path.read_text(encoding='utf-8')) + if not isinstance(bak_data, dict): + raise ValueError("backup payload is not a session object") + from api.models import _repair_session_message_projections + + bak_data, _, _ = _repair_session_message_projections(bak_data) + bak_messages = bak_data.get('messages') + if isinstance(bak_messages, list): + bak_data['message_count'] = len(bak_messages) + tmp_path.write_text( + json.dumps(bak_data, ensure_ascii=False, indent=2), encoding='utf-8' + ) tmp_path.replace(session_path) - except OSError as exc: + except (OSError, json.JSONDecodeError, ValueError) as exc: logger.warning("recover_session: copy failed for %s: %s", session_path, exc) try: tmp_path.unlink(missing_ok=True) @@ -551,13 +579,24 @@ def _read_state_db_missing_sidecar_rows( if {'session_id', 'role', 'content'}.issubset(message_cols): order = "timestamp, id" if 'timestamp' in message_cols and 'id' in message_cols else "rowid" ts_expr = 'timestamp' if 'timestamp' in message_cols else 'NULL AS timestamp' + # A recovered sidecar needs durable per-row provenance before + # the normal Session.load/save replay reducers run. Without + # it, two legitimate state.db rows with identical role, + # content, and timestamp collapse irreversibly on first load. + row_id_expr = ( + 'id AS _state_db_row_id' + if 'id' in message_cols + else 'rowid AS _state_db_row_id' + ) for msg in conn.execute( - f"SELECT role, content, {ts_expr} FROM messages WHERE session_id = ? ORDER BY {order}", + f"SELECT role, content, {ts_expr}, {row_id_expr} " + f"FROM messages WHERE session_id = ? ORDER BY {order}", (sid,), ).fetchall(): message = { 'role': msg['role'], 'content': msg['content'] or '', + '_state_db_row_id': msg['_state_db_row_id'], } if msg['timestamp'] is not None: message['timestamp'] = msg['timestamp'] diff --git a/api/streaming.py b/api/streaming.py index 489c91a8721..c1255d8f627 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -22,6 +22,8 @@ import traceback import copy import inspect +from bisect import bisect_left +from collections import Counter from pathlib import Path from typing import Optional @@ -76,10 +78,20 @@ from api.models import ( StateDBSessionMessagesSnapshot, _WEBUI_TRUSTED_AGENT_INPUT_FIELD, - _is_empty_partial_activity_message, _message_exact_timestamp_details, _message_private_identity_compatible, _validated_webui_pending_user_timestamp_identity, + _SESSION_MESSAGE_DISPLAY_METADATA_KEYS, + _STRUCTURED_REPLAY_FIELDS, + _canonical_message_digest, + _collapse_replayed_assistant_rows, + _durable_empty_assistant_replay_key, + _incomplete_reasoning_message_id, + _is_admissible_empty_text_content, + _is_empty_partial_activity_message, + _message_has_structured_replay_fields, + _partial_message_signature as _durable_partial_message_signature, + _strict_incomplete_message_id_key, _evict_sessions_over_cap, clear_process_wakeup_pause, get_state_db_session_messages, @@ -1999,13 +2011,15 @@ def _active_turn_authority(session, stream_id, msg_text): def _coerce_current_turn_user_idx(value): - if isinstance(value, bool) or value is None: + if type(value) is not int or value < 0: return None - try: - idx = int(value) - except (TypeError, ValueError): - return None - return idx if idx >= 0 else None + return value + + +def _coerce_current_turn_id(value): + if type(value) is not str: + return '' + return value.strip() def _resolve_active_turn_authority(identity, *, result=None, agent=None): @@ -2014,6 +2028,8 @@ def _resolve_active_turn_authority(identity, *, result=None, agent=None): resolved = dict(identity) resolved.pop('agent_turn_boundary_resolved', None) resolved.pop('agent_turn_boundary_source', None) + resolved['current_turn_user_idx'] = None + resolved['turn_id'] = '' # Treat the boundary as one coherent pair. In particular, do not retain the # failed Agent instance's index/turn when credential self-heal creates a @@ -2025,7 +2041,7 @@ def _resolve_active_turn_authority(identity, *, result=None, agent=None): _boundary_source = '' if isinstance(result, dict): _result_idx = _coerce_current_turn_user_idx(result.get('current_turn_user_idx')) - _result_turn_id = str(result.get('turn_id') or '').strip() + _result_turn_id = _coerce_current_turn_id(result.get('turn_id')) if _result_idx is not None and _result_turn_id: _boundary_idx = _result_idx _boundary_turn_id = _result_turn_id @@ -2034,7 +2050,9 @@ def _resolve_active_turn_authority(identity, *, result=None, agent=None): _agent_idx = _coerce_current_turn_user_idx( getattr(agent, '_persist_user_message_idx', None) ) - _agent_turn_id = str(getattr(agent, '_current_turn_id', '') or '').strip() + _agent_turn_id = _coerce_current_turn_id( + getattr(agent, '_current_turn_id', '') + ) if _agent_idx is not None and _agent_turn_id and not _boundary_source: _boundary_idx = _agent_idx _boundary_turn_id = _agent_turn_id @@ -2050,11 +2068,138 @@ def _resolve_active_turn_authority(identity, *, result=None, agent=None): def _active_turn_boundary_is_valid(identity): if not isinstance(identity, dict): return False - if not str(identity.get('turn_id') or '').strip(): + turn_id = identity.get('turn_id') + if type(turn_id) is not str or not turn_id.strip(): return False + current_turn_user_idx = identity.get('current_turn_user_idx') return ( identity.get('agent_turn_boundary_resolved') is True - and isinstance(identity.get('current_turn_user_idx'), int) + and type(current_turn_user_idx) is int + and current_turn_user_idx >= 0 + ) + + +def _exact_out_of_band_history_size(result_messages, previous_context): + """Return ``len(previous_context)`` when the result is exact history + delta. + + Conclusive only when every later row is assistant/tool output (or a + compression marker), the durable context contains a real user row, and the + result prefix equals the complete context under a strict canonical digest: + either the persisted rows themselves, or their exact Agent-bound projection + (``_sanitize_messages_for_agent``), which is what the Agent receives and + replays. The projection strips WebUI-only metadata (ids, timestamps, turn + tokens) but keeps every provider-facing field, so visible-text lookalikes + and payload-distinct rows still fail closed (``None``). + """ + result_messages = list(result_messages or []) + previous_context = list(previous_context or []) + history_size = len(previous_context) + delta = result_messages[history_size:] + if not ( + previous_context + and delta + and any( + type(message) is dict and message.get('role') == 'user' + for message in previous_context + ) + and all( + _is_context_compression_marker(message) + or ( + type(message) is dict + and message.get('role') in ('assistant', 'tool') + ) + for message in delta + ) + ): + return None + if _messages_have_prefix( + result_messages, + previous_context, + key_fn=_canonical_replay_digest, + ): + return history_size + try: + agent_bound = _sanitize_messages_for_agent(previous_context) + except Exception: + return None + # A projection that drops or merges rows cannot be aligned by index with + # the durable context; fail closed instead of guessing the boundary. + if len(agent_bound) != history_size: + return None + for replayed, projected in zip(result_messages[:history_size], agent_bound, strict=True): + replayed_key = _canonical_replay_digest(replayed) + if replayed_key is None or replayed_key != _canonical_replay_digest(projected): + return None + return history_size + + +def _result_has_authoritative_full_history_prefix( + result_messages, + previous_context, + identity, + msg_text, +): + """Prove that an exact result prefix is history, not a lookalike delta.""" + result_messages = list(result_messages or []) + previous_context = list(previous_context or []) + # Even when an older Agent cannot expose its current-turn index/turn id, a + # complete byte-structural replay of the durable context followed only by + # assistant/tool output is conclusive history. This is deliberately the + # canonical digest comparator, not visible-text equivalence: payload- + # distinct lookalikes still fail closed and are preserved. + if _exact_out_of_band_history_size(result_messages, previous_context) is not None: + return True + if not _active_turn_boundary_is_valid(identity): + return False + current_turn_user_idx = identity['current_turn_user_idx'] + if current_turn_user_idx < 0 or current_turn_user_idx > len(previous_context): + return False + if current_turn_user_idx == len(previous_context): + # The Agent can omit the separately supplied current user while returning + # exact prior history plus assistant/tool output. Require a real historical + # user so an assistant-only structured lookalike delta still fails closed. + has_historical_user = any( + type(message) is dict and message.get('role') == 'user' + for message in previous_context + ) + out_of_band_delta = result_messages[current_turn_user_idx:] + if has_historical_user and out_of_band_delta and all( + _is_context_compression_marker(message) + or ( + type(message) is dict + and message.get('role') in ('assistant', 'tool') + ) + for message in out_of_band_delta + ): + return _messages_have_prefix( + result_messages, + previous_context, + key_fn=_canonical_replay_digest, + ) + if current_turn_user_idx >= len(result_messages): + return False + current_turn = result_messages[current_turn_user_idx] + expected_text = identity.get('text') if identity.get('text') is not None else msg_text + if ( + type(current_turn) is not dict + or current_turn.get('role') != 'user' + or _normalize_user_text(current_turn.get('content')) + != _normalize_user_text(expected_text) + ): + return False + if current_turn_user_idx < len(previous_context): + previous_current_turn = previous_context[current_turn_user_idx] + if ( + type(previous_current_turn) is not dict + or previous_current_turn.get('role') != 'user' + or _normalize_user_text(previous_current_turn.get('content')) + != _normalize_user_text(expected_text) + ): + return False + return _messages_have_prefix( + result_messages, + previous_context, + key_fn=_canonical_replay_digest, ) @@ -2171,10 +2316,17 @@ def _owner_projection_current_turn_row(messages, identity): return None -def _find_active_turn_checkpoint_index(result_messages, previous_context, identity, msg_text): +def _find_active_turn_checkpoint_index( + result_messages, + previous_context, + identity, + msg_text, + *, + allow_exact_prefix=False, +): """Locate the current turn's user row inside ``result_messages``. - Exactly one declared index domain is supported. The WebUI token is the + Exactly one Agent-declared index domain is supported. The WebUI token is the strongest proof and wins when it survives the Agent projection. Otherwise the Agent-resolved ``current_turn_user_idx`` addresses ``result["messages"]`` directly, so only that exact index is validated. With a repeated prompt a @@ -2184,10 +2336,10 @@ def _find_active_turn_checkpoint_index(result_messages, previous_context, identi shifted projection is ever genuinely required, the Agent must carry an explicit projection-origin/base-length discriminator instead. - ``previous_context`` is retained for call-site compatibility only; it does - not participate in index resolution. + A legacy result without Agent provenance is accepted only at the boundary + following a complete canonical replay of ``previous_context``. No shifted + or alternate Agent-index projection is ever probed. """ - del previous_context # single declared index domain: result_messages only result_messages = list(result_messages or []) if not isinstance(identity, dict): return None @@ -2195,8 +2347,29 @@ def _find_active_turn_checkpoint_index(result_messages, previous_context, identi for idx, message in enumerate(result_messages): if _active_turn_token_matches(message, identity): return idx + expected_text = identity.get('text') if identity.get('text') is not None else msg_text if not _active_turn_boundary_is_valid(identity): + previous_context = list(previous_context or []) + legacy_idx = len(previous_context) + if ( + allow_exact_prefix + and legacy_idx < len(result_messages) + and _messages_have_prefix( + result_messages, + previous_context, + key_fn=_canonical_replay_digest, + ) + ): + message = result_messages[legacy_idx] + if ( + isinstance(message, dict) + and message.get('role') == 'user' + and _normalize_user_text(_message_text(message.get('content'))) + == _normalize_user_text(expected_text) + ): + return legacy_idx return None + del previous_context, allow_exact_prefix idx = identity['current_turn_user_idx'] if idx < 0 or idx >= len(result_messages): return None @@ -2285,7 +2458,15 @@ def _materialize_active_turn_user(identity, msg_text, source): return message -def _settle_current_turn_boundary(previous_context, result_messages, identity, msg_text, source): +def _settle_current_turn_boundary( + previous_context, + result_messages, + identity, + msg_text, + source, + *, + allow_exact_prefix=False, +): """Insert the pending turn before assistant/tool output when it is absent.""" result_messages = list(result_messages or []) if not result_messages or not isinstance(identity, dict): @@ -2295,6 +2476,7 @@ def _settle_current_turn_boundary(previous_context, result_messages, identity, m previous_context, identity, msg_text, + allow_exact_prefix=allow_exact_prefix, ) if _checkpoint_idx is not None: existing_checkpoint = result_messages[_checkpoint_idx] @@ -2313,7 +2495,11 @@ def _settle_current_turn_boundary(previous_context, result_messages, identity, m ) return result_messages previous_context = list(previous_context or []) - if _messages_have_prefix(result_messages, previous_context): + if _messages_have_prefix( + result_messages, + previous_context, + allow_exact_payload=allow_exact_prefix, + ): insert_at = len(previous_context) elif _active_turn_boundary_is_valid(identity): insert_at = identity['current_turn_user_idx'] - len(previous_context) @@ -2340,12 +2526,98 @@ def _settle_current_turn_boundary(previous_context, result_messages, identity, m ) +def _share_active_turn_checkpoint_id(result_messages, context_messages, identity): + """Keep the token-owned user row on one stable id across projections. + + Payload-distinct prefix reconciliation intentionally deep-copies provider + rows before stable ids are minted. The request-local active-turn token is + the explicit authority that lets us reconnect only the current user row; + no visible-text or timestamp inference is involved. + """ + if not isinstance(identity, dict) or not identity.get('token'): + return + result_row = next( + ( + message + for message in result_messages or [] + if _active_turn_token_matches(message, identity) + ), + None, + ) + context_row = next( + ( + message + for message in context_messages or [] + if _active_turn_token_matches(message, identity) + ), + None, + ) + if context_row is None: + return + + result_id_key = ( + _strict_incomplete_message_id_key(result_row.get('id')) + if isinstance(result_row, dict) + else None + ) + context_id_key = _strict_incomplete_message_id_key(context_row.get('id')) + if result_id_key is not None and context_row.get('id') is None: + context_row['id'] = result_row['id'] + return + if context_id_key is not None and isinstance(result_row, dict) and result_row.get('id') is None: + result_row['id'] = context_row['id'] + return + if result_id_key is None and context_id_key is None and context_row.get('id') is None: + _assign_stable_message_ids( + [context_row], + result_messages, + context_messages, + ) + if isinstance(result_row, dict) and result_row.get('id') is None: + result_row['id'] = context_row['id'] + + def _align_current_turn_display(previous_display, previous_context, identity): """Make a context-only exact checkpoint visible before shared settlement.""" display = list(previous_display or []) context = list(previous_context or []) if not isinstance(identity, dict): return display, context + # Legacy/eager checkpoints may predate the request-local token. Stamp the + # exact indexed context row only when its pending-turn timestamp also + # matches; index + visible text alone can point at an older identical prompt + # and would retoken historical data. Existing token matches remain valid. + context, _ = _mark_active_turn_checkpoint_in_history( + context, + identity, + identity.get('text'), + allow_index_fallback=False, + ) + legacy_checkpoint = None + if not _active_turn_has_checkpoint(context, identity): + idx = _coerce_current_turn_user_idx(identity.get('current_turn_user_idx')) + candidate = context[idx] if idx is not None and idx < len(context) else None + candidate_timestamp = ( + candidate.get('timestamp', candidate.get('_ts')) + if isinstance(candidate, dict) + else None + ) + identity_timestamp = identity.get('timestamp') + if ( + isinstance(candidate, dict) + and candidate.get('role') == 'user' + and _normalize_user_text(candidate.get('content')) + == _normalize_user_text(identity.get('text')) + and isinstance(candidate_timestamp, (int, float)) + and isinstance(identity_timestamp, (int, float)) + and float(candidate_timestamp) == float(identity_timestamp) + ): + if _active_turn_boundary_is_valid(identity): + _mark_active_turn_checkpoint(candidate, identity) + elif not candidate.get('_active_turn_token'): + # Preserve an untokened legacy checkpoint as the visible row, + # but do not relabel it as token-owned without Agent authority. + legacy_checkpoint = copy.deepcopy(candidate) display, _ = _mark_active_turn_checkpoint_in_history( display, identity, @@ -2358,6 +2630,8 @@ def _align_current_turn_display(previous_display, previous_context, identity): context, identity, ) + if checkpoint is None and legacy_checkpoint is not None: + checkpoint = legacy_checkpoint if checkpoint is None and identity.get('token'): checkpoint = _materialize_active_turn_user( identity, @@ -2405,6 +2679,25 @@ def _prepare_marker_clean_writeback( boundary = _active_turn_boundary( cleaned, previous_context_messages, active_turn_identity, msg_text, ) + # An exact replay of the whole durable context followed only by + # assistant/tool output proves every replayed row is history, even when + # a repeated prompt leaves the visible-text fallback at boundary 0. + # Otherwise display restoration would treat the replayed history as + # current output while model context keeps the answer. Exact Agent + # authority placing the current user inside that history still wins. + exact_history_size = _exact_out_of_band_history_size( + cleaned, previous_context_messages, + ) + if ( + exact_history_size is not None + and boundary < exact_history_size + and not ( + _active_turn_boundary_is_valid(active_turn_identity) + and active_turn_identity['current_turn_user_idx'] + < exact_history_size + ) + ): + boundary = exact_history_size return ( cleaned, _restore_reasoning_metadata_before_boundary( @@ -2413,7 +2706,14 @@ def _prepare_marker_clean_writeback( provenance, boundary, ) - return [], list(previous_context_messages or []), provenance, 0 + # A non-empty Agent result can consist entirely of synthetic verify-loop + # controls. The display merge still runs its canonical replay repair in + # that shape, so apply the same durable pipeline to context instead of + # carrying pre-existing exact assistant replays until a later save/load. + repaired_context, _ = _collapse_replayed_assistant_rows( + previous_context_messages + ) + return [], repaired_context, provenance, 0 def _annotate_media_snapshots_for_settled_messages(messages) -> None: @@ -2434,6 +2734,187 @@ def _annotate_media_snapshots_for_settled_messages(messages) -> None: logger.debug("Media snapshot annotation failed during settle", exc_info=True) +def _has_replay_safe_history_prefix(messages, previous_context) -> bool: + """Return whether the complete prior context is a conclusive replay prefix.""" + previous_context = list(previous_context or []) + return bool( + previous_context + and _messages_have_prefix( + messages, + previous_context, + key_fn=_canonical_replay_digest, + ) + ) + + +def _reconcile_payload_distinct_history_prefix( + previous_context, + result_messages, + identity, + msg_text, + source, +): + """Preserve a positionally historical prefix rejected by strict equality. + + The Agent's current-turn index can prove that the first ``len(previous)`` + rows precede an out-of-band user turn without proving those rows are replay + duplicates. Keep the durable previous projection, materialize the current + user boundary, then carry every payload-distinct returned prefix row as + current-turn delta. The rebuilt list has an exact previous prefix, so + downstream reconciliation never needs a weaker visible-text fallback. + """ + previous_context = list(previous_context or []) + result_messages = list(result_messages or []) + if ( + not previous_context + or not _active_turn_boundary_is_valid(identity) + or identity['current_turn_user_idx'] != len(previous_context) + or len(result_messages) < len(previous_context) + or not any( + type(message) is dict and message.get('role') == 'user' + for message in previous_context + ) + ): + return result_messages, False + + expected_text = identity.get('text') if identity.get('text') is not None else msg_text + suffix = list(result_messages[len(previous_context):]) + current_user_echoed = bool( + suffix + and type(suffix[0]) is dict + and suffix[0].get('role') == 'user' + and _normalize_user_text(suffix[0].get('content')) + == _normalize_user_text(expected_text) + ) + out_of_band_suffix = bool(suffix) and all( + _is_context_compression_marker(message) + or ( + type(message) is dict + and message.get('role') in ('assistant', 'tool') + ) + for message in suffix + ) + if not current_user_echoed and not out_of_band_suffix: + return result_messages, False + + payload_distinct_prefix_rows = [] + for actual, durable in zip( + result_messages[:len(previous_context)], + previous_context, + strict=True, + ): + if _comparison_keys_equal( + _canonical_replay_digest(actual), + _canonical_replay_digest(durable), + ): + continue + payload_distinct_prefix_rows.append(copy.deepcopy(actual)) + + if current_user_echoed: + suffix = copy.deepcopy(suffix) + current_user = suffix.pop(0) + _mark_active_turn_checkpoint(current_user, identity) + else: + current_user = _materialize_active_turn_user(identity, msg_text, source) + suffix = copy.deepcopy(suffix) + return ( + copy.deepcopy(previous_context) + + [current_user] + + payload_distinct_prefix_rows + + suffix, + True, + ) + + +def _collapse_replays_with_history_boundary( + messages, + previous_context, + *, + history_prefix_is_authoritative=False, +): + """Reduce exact replays without crossing a proven history/delta boundary. + + The Agent can return a full history without echoing the separately supplied + current user message. In that shape the previous assistant and the new + assistant are adjacent even though they belong to different user turns. A + whole-list reduction before stable IDs are assigned would erase a legitimate + repeated answer. When the complete prior context is a replay-safe prefix, + reduce the historical prefix and current delta independently; otherwise fail + closed to the existing whole-list strict reducer. + """ + messages = list(messages or []) + previous_context = list(previous_context or []) + if ( + history_prefix_is_authoritative is True + and _has_replay_safe_history_prefix(messages, previous_context) + ): + boundary = len(previous_context) + history, history_changed = _collapse_replayed_assistant_rows( + messages[:boundary] + ) + delta, delta_changed = _collapse_replayed_assistant_rows(messages[boundary:]) + return history + delta, bool(history_changed or delta_changed) + return _collapse_replayed_assistant_rows(messages) + + +def _collapse_repeated_exact_history_prefixes( + previous_context, + result_messages, + msg_text, +): + """Reduce repeated full-history blocks before strict prefix classification. + + Legacy Agent implementations can prepend the complete context more than + once, then append an explicit current-user row. Only exact canonical block + copies are removable here: payload-distinct lookalikes remain incomparable, + and an explicit matching user boundary is required so an assistant-only + current-turn delta cannot be mistaken for another history replay. + """ + previous_context = list(previous_context or []) + result_messages = list(result_messages or []) + block_size = len(previous_context) + if block_size == 0 or len(result_messages) < (2 * block_size) + 1: + return result_messages, False + + history_keys = [ + _canonical_replay_digest(message) for message in previous_context + ] + if any(key is None for key in history_keys): + return result_messages, False + + def _block_matches(start): + if start + block_size > len(result_messages): + return False + return all( + _comparison_keys_equal( + _canonical_replay_digest(actual), + expected, + ) + for actual, expected in zip( + result_messages[start:start + block_size], + history_keys, + strict=True, + ) + ) + + if not _block_matches(0) or not _block_matches(block_size): + return result_messages, False + + cursor = block_size + while _block_matches(cursor): + cursor += block_size + suffix = result_messages[cursor:] + if not ( + suffix + and type(suffix[0]) is dict + and suffix[0].get('role') == 'user' + and _normalize_user_text(suffix[0].get('content')) + == _normalize_user_text(msg_text) + ): + return result_messages, False + return copy.deepcopy(previous_context) + copy.deepcopy(suffix), True + + def _settle_result_messages( session, previous_messages, @@ -2443,6 +2924,22 @@ def _settle_result_messages( source, active_turn_identity, ): + result_messages, repeated_exact_history_prefix = ( + _collapse_repeated_exact_history_prefixes( + previous_context_messages, + result_messages, + msg_text, + ) + ) + result_has_authoritative_full_history_prefix = ( + repeated_exact_history_prefix + or _result_has_authoritative_full_history_prefix( + result_messages, + previous_context_messages, + active_turn_identity, + msg_text, + ) + ) ( result_messages, next_context_messages, @@ -2455,6 +2952,72 @@ def _settle_result_messages( msg_text, ) if result_messages: + if not result_has_authoritative_full_history_prefix: + reconciled_result, result_reconciled = ( + _reconcile_payload_distinct_history_prefix( + previous_context_messages, + result_messages, + active_turn_identity, + msg_text, + source, + ) + ) + reconciled_context, context_reconciled = ( + _reconcile_payload_distinct_history_prefix( + previous_context_messages, + next_context_messages, + active_turn_identity, + msg_text, + source, + ) + ) + if result_reconciled and context_reconciled: + result_messages = reconciled_result + next_context_messages = reconciled_context + result_has_authoritative_full_history_prefix = True + # Classify exact adjacent replays before generated stable IDs make two + # source-identical assistant rows artificially distinct. Keep a proven + # prior-history prefix separate from its delta: the current user message + # is supplied out of band, so equal assistants on that boundary are two + # legitimate turns rather than an adjacent replay. + result_messages, _ = _collapse_replays_with_history_boundary( + result_messages, + previous_context_messages, + history_prefix_is_authoritative=( + result_has_authoritative_full_history_prefix + ), + ) + next_context_messages, _ = _collapse_replays_with_history_boundary( + next_context_messages, + previous_context_messages, + history_prefix_is_authoritative=( + result_has_authoritative_full_history_prefix + ), + ) + replay_safe_history_prefix = bool( + result_has_authoritative_full_history_prefix + and _has_replay_safe_history_prefix( + next_context_messages, + previous_context_messages, + ) + ) + if ( + result_has_authoritative_full_history_prefix + or replay_safe_history_prefix + ): + # Establish the out-of-band user-turn boundary while the exact + # source prefix is still intact. Generated stable IDs intentionally + # mutate only result/context rows, so any later comparison with the + # original idless context would lose this already-proven authority. + # Inconclusive structured lookalikes still skip this path. + next_context_messages = _settle_current_turn_boundary( + previous_context_messages, + next_context_messages, + active_turn_identity, + msg_text, + source, + allow_exact_prefix=True, + ) _assign_stable_message_ids( result_messages, previous_messages, @@ -2464,6 +3027,9 @@ def _settle_result_messages( previous_context_messages, next_context_messages, msg_text, + result_prefix_is_authoritative=( + result_has_authoritative_full_history_prefix + ), ) next_context_messages = _settle_current_turn_boundary( previous_context_messages, @@ -2471,12 +3037,28 @@ def _settle_result_messages( active_turn_identity, msg_text, source, + allow_exact_prefix=True, + ) + if ( + result_messages + and result_has_authoritative_full_history_prefix + and _messages_have_prefix( + next_context_messages, + previous_context_messages, + key_fn=_canonical_replay_digest, + ) + ): + history_size = len(previous_context_messages) + session.context_messages = ( + copy.deepcopy(previous_context_messages) + + _deduplicate_context_messages(next_context_messages[history_size:]) + ) + else: + session.context_messages = ( + _deduplicate_context_messages(next_context_messages) + if result_messages + else list(next_context_messages or []) ) - session.context_messages = ( - _deduplicate_context_messages(next_context_messages) - if result_messages - else list(next_context_messages or []) - ) if result_messages: session.context_messages = _settle_current_turn_boundary( previous_context_messages, @@ -2484,7 +3066,13 @@ def _settle_result_messages( active_turn_identity, msg_text, source, + allow_exact_prefix=True, ) + _share_active_turn_checkpoint_id( + result_messages, + session.context_messages, + active_turn_identity, + ) previous_display_for_writeback, session.context_messages = _align_current_turn_display( previous_messages, session.context_messages, @@ -2499,6 +3087,9 @@ def _settle_result_messages( msg_text, source=source, verification_nudge_provenance=verification_nudge_provenance, + result_has_authoritative_full_history_prefix=( + result_has_authoritative_full_history_prefix + ), ) trusted_agent_input = ( active_turn_identity.get('trusted_agent_input_text') @@ -6375,7 +6966,7 @@ def _api_safe_message_positions(messages): def _deduplicate_context_messages(messages): - """Remove duplicate messages from context by identity, keeping first occurrence. + """Remove only replay-safe duplicates from provider-facing context. Prevents the agent from seeing the same message twice in conversation_history when result_messages contain duplicates that weren't caught by display-merge. @@ -6405,6 +6996,12 @@ def _deduplicate_context_messages(messages): if _is_compressed_context_tool_result_summary_message(msg) and not msg.get('tool_call_id'): deduped.append(msg) continue + if type(msg) is dict and msg.get('role') == 'assistant': + # Assistant payloads are provider-owned. Preserve them here and run + # the same exact replay pipeline used by display and persistence + # after user/marker normalization has established adjacency. + deduped.append(msg) + continue # Context ownership is provider-facing: two rows with identical visible # text but different durable ``api_content`` sidecars are distinct turns. # Keep the display identity unchanged so ordinary transcript dedup still @@ -6421,6 +7018,20 @@ def _deduplicate_context_messages(messages): prior_exact_idx = user_exact_index.get(user_exact_key) if msg.get('_active_turn_token'): if prior_exact_idx is not None: + prior_row = deduped[prior_exact_idx] + prior_token = ( + prior_row.get('_active_turn_token') + if isinstance(prior_row, dict) + else None + ) + if prior_token and prior_token != msg.get('_active_turn_token'): + # Conflicting request-local turn tokens prove two + # distinct durable turns that merely share a visible + # projection. Replacing would delete history; keep + # both rows. + deduped.append(msg) + user_exact_index[user_exact_key] = len(deduped) - 1 + continue deduped[prior_exact_idx] = msg continue if key in seen: @@ -6435,6 +7046,7 @@ def _deduplicate_context_messages(messages): if key is not None: seen.add(key) deduped.append(msg) + deduped, _ = _collapse_replayed_assistant_rows(deduped) return deduped @@ -6830,13 +7442,40 @@ def _message_identity(msg): # Now, _partial messages with empty text get a stable identity # keyed on their role + _partial flag + reasoning/tool metadata, # so the merge can dedup identical empty partials. + # Codex can persist a reasoning-only assistant result with an empty + # visible body and finish_reason=incomplete without the legacy + # ``_partial`` flag. Those rows still carry the stable core message id. + # Returning None here made every reconcile treat the same result as a + # fresh context-only row, which amplified alternating replays such as + # FD05 message ids 1701/1702 on every subsequent turn. + # #6600: share the persistence boundary's strict typed scalar identity + # (api.models._strict_incomplete_message_id_key) so str/int/float ids + # never collapse across types and bools/containers/subclasses/non-finite + # floats are rejected in BOTH layers. + if ( + role == 'assistant' + and str(msg.get('finish_reason') or '').lower() == 'incomplete' + ): + typed_id_key = _incomplete_reasoning_message_id(msg) + if typed_id_key is not None: + return ( + role, + '', + '', + '__incomplete_message_id__' + repr(typed_id_key), + ) + return None + # Canonical incomplete identity must win over the legacy partial arm: + # persistence keys `_partial + incomplete` rows by typed message id too. if msg.get('_partial'): - reasoning_key = " ".join(str(msg.get('reasoning') or '').split())[:200] + partial_digest = _durable_partial_message_signature(msg) + if partial_digest is None: + return None return ( role, '', # empty text '', # no tool_call_id - '__partial__' + reasoning_key, + '__partial__' + partial_digest.hex(), ) return None return ( @@ -6847,18 +7486,240 @@ def _message_identity(msg): ) -def _messages_have_prefix(messages, prefix, *, key_fn=None): +def _comparison_keys_equal(left, right): + """Return True only when both comparison keys are conclusive and equal.""" + return left is not None and right is not None and left == right + + +def _canonical_replay_digest(message): + """Strict canonical payload digest for replay/prefix comparison. + + ``_active_turn_token`` is request-local bookkeeping, not payload: + ``_sanitize_messages_for_agent()`` strips it from the history the Agent + replays back, so a persisted row and its exact replayed copy can differ by + that one field only. Exclude it from the comparison digest on both sides; + every other byte of the payload must still match exactly, and non-strict + JSON payloads keep failing closed to ``None``. + """ + if type(message) is dict and '_active_turn_token' in message: + message = { + key: value + for key, value in message.items() + if key != '_active_turn_token' + } + return _canonical_message_digest(message) + + +def _display_backfill_key(message): + """Return one strict unary backfill identity. + + The actual display/context join is prepared by + :func:`_display_backfill_projection_keys`, which can safely resolve the + one-sided-id case without making equality non-transitive. This unary form is + exact when both rows carry a compatible stable id or both lack one. + """ + details = _display_backfill_identity_details(message) + if details is None: + return None + stable_id, visible_digest, common_digest = details + if stable_id is not None: + return ('stable_row', stable_id, visible_digest) + return ('common_projection', common_digest) + + +def _display_backfill_identity_details(message): + """Return typed stable id plus visible/common strict digests. + + Display-only settlement enrichment is excluded from the common projection; + every unknown field remains durable and therefore identity-bearing. Stable + id aliases are type-faithful and contradictory or non-JSON scalar aliases + are incomparable rather than coerced. + """ + if type(message) is not dict: + return None + + typed_ids = set() + has_untyped_id = False + for key in ('id', 'message_id'): + value = message.get(key) if key in message else None + if value is None or (type(value) is str and value == ''): + continue + typed_id = _strict_incomplete_message_id_key(value) + if typed_id is None: + # A bool/container id carries no stable identity, but the row is + # still comparable: keep the raw id inside the type-faithful common + # digest below, so only a byte-identical copy (``True`` never + # equals ``1``) can join it across projections. + has_untyped_id = True + continue + typed_ids.add(typed_id) + if len(typed_ids) > 1 or (typed_ids and has_untyped_id): + return None + stable_id = next(iter(typed_ids)) if typed_ids else None + excluded_id_keys = set() if has_untyped_id else {'id', 'message_id'} + + visible_digest = _canonical_message_digest( + { + 'role': message.get('role'), + 'content': message.get('content'), + } + ) + common_digest = _canonical_message_digest( + { + key: value + for key, value in message.items() + if key not in _SESSION_MESSAGE_DISPLAY_METADATA_KEYS + and key not in excluded_id_keys + } + ) + if visible_digest is None or common_digest is None: + return None + return stable_id, visible_digest, common_digest + + +def _display_backfill_projection_keys(previous_display, previous_context): + """Build transitive cross-projection keys without payload-weak guesses. + + A compatible typed stable id is authoritative when both rows carry it. If + one projection lacks the id, a strict common payload may bridge it only when + that payload maps to one stable id across both projections. Ambiguous or + malformed rows receive projection-local keys, so they are preserved instead + of arbitrarily paired. + """ + display_details = [ + _display_backfill_identity_details(message) + for message in previous_display + ] + context_details = [ + _display_backfill_identity_details(message) + for message in previous_context + ] + stable_ids_by_common = {} + for details in (*display_details, *context_details): + if details is None: + continue + stable_id, _visible_digest, common_digest = details + if stable_id is not None: + stable_ids_by_common.setdefault(common_digest, set()).add(stable_id) + + def _projection_key(details, projection, index): + if details is None: + return ('incomparable', projection, index) + stable_id, visible_digest, common_digest = details + if stable_id is not None: + return ('stable_row', stable_id, visible_digest) + candidate_ids = stable_ids_by_common.get(common_digest, set()) + if len(candidate_ids) == 1: + return ('stable_row', next(iter(candidate_ids)), visible_digest) + if candidate_ids: + return ('incomparable', projection, index) + return ('common_projection', common_digest) + + return ( + [ + _projection_key(details, 'display', index) + for index, details in enumerate(display_details) + ], + [ + _projection_key(details, 'context', index) + for index, details in enumerate(context_details) + ], + ) + + +def _message_content_has_nontext_parts(content) -> bool: + """Return True when visible-text identity would discard content structure.""" + if type(content) is not list: + return type(content) not in (str, type(None)) + for part in content: + if type(part) is not dict: + return True + part_type = part.get('type') + if type(part_type) is not str: + return True + if part_type.lower() not in ('', 'text', 'input_text', 'output_text'): + return True + return False + + +def _structured_replay_value_is_nonempty(value) -> bool: + """Classify empty JSON containers without invoking foreign equality hooks.""" + if value is None: + return False + if type(value) in (str, list, dict): + return bool(value) + return True + + +def _message_requires_exact_prefix_payload(message) -> bool: + """Return True when coarse visible identity is destructive for a prefix.""" + if type(message) is not dict: + return True + if type(message.get('content', '')) not in (str, type(None)): + return True + return bool( + any( + _structured_replay_value_is_nonempty(message.get(field)) + for field in _STRUCTURED_REPLAY_FIELDS + ) + or _structured_replay_value_is_nonempty(message.get('api_content')) + ) + + +def _messages_have_prefix( + messages, + prefix, + *, + key_fn=None, + allow_exact_payload=False, +): + strict_keys_only = key_fn is not None key_fn = key_fn or _message_identity if len(messages or []) < len(prefix or []): return False for idx, expected in enumerate(prefix or []): - if key_fn((messages or [])[idx]) != key_fn(expected): + actual = (messages or [])[idx] + if strict_keys_only: + if _comparison_keys_equal(key_fn(actual), key_fn(expected)): + continue return False + if ( + _message_requires_exact_prefix_payload(actual) + or _message_requires_exact_prefix_payload(expected) + ): + if allow_exact_payload and _comparison_keys_equal( + _canonical_replay_digest(actual), + _canonical_replay_digest(expected), + ): + continue + return False + if _comparison_keys_equal(key_fn(actual), key_fn(expected)): + continue + if allow_exact_payload and _comparison_keys_equal( + _canonical_replay_digest(actual), + _canonical_replay_digest(expected), + ): + continue + return False return True def _message_replay_key(msg): """Return a stable comparison key for replay/overlap de-duplication.""" + durable_empty_key = _durable_empty_assistant_replay_key(msg) + if durable_empty_key is not None: + return ('durable_empty', durable_empty_key) + if type(msg) is not dict: + return None + if ( + _message_has_structured_replay_fields(msg) + or _message_content_has_nontext_parts(msg.get('content', '')) + or any( + _structured_replay_value_is_nonempty(msg.get(field)) + for field in _STRUCTURED_REPLAY_FIELDS + ) + ): + return None identity = _message_identity(msg) # ``api_content`` is a provider-facing replay sidecar. It must participate # in context/replay overlap identity or two same-visible turns can collapse @@ -6875,7 +7736,10 @@ def _message_replay_key(msg): if sidecar is not None: return (*identity, sidecar) return identity - if not isinstance(msg, dict): + if ( + str(msg.get('role') or '') == 'assistant' + and not _is_admissible_empty_text_content(msg.get('content')) + ): return None key = ( str(msg.get('role') or ''), @@ -6886,7 +7750,12 @@ def _message_replay_key(msg): return (*key, sidecar) if sidecar is not None else key -def _strip_replayed_prefix(existing_messages, candidates): +def _strip_replayed_prefix( + existing_messages, + candidates, + *, + key_fn=_message_replay_key, +): """Drop a candidate prefix that is already the suffix of existing_messages. Compression/continuation can replay the active tail from state.db after the @@ -6898,9 +7767,12 @@ def _strip_replayed_prefix(existing_messages, candidates): candidates = list(candidates or []) max_overlap = min(len(existing_messages), len(candidates)) for overlap in range(max_overlap, 0, -1): - left = [_message_replay_key(m) for m in existing_messages[-overlap:]] - right = [_message_replay_key(m) for m in candidates[:overlap]] - if left == right: + left = [key_fn(m) for m in existing_messages[-overlap:]] + right = [key_fn(m) for m in candidates[:overlap]] + if all( + _comparison_keys_equal(left_key, right_key) + for left_key, right_key in zip(left, right, strict=True) + ): return candidates[overlap:] return candidates @@ -6947,8 +7819,12 @@ def _strip_replayed_context_items(existing_messages, candidates): if not existing_messages or not candidates: return candidates - existing_keys = [_message_replay_key(m) for m in existing_messages] - candidate_keys = [_message_replay_key(m) for m in candidates] + # This is a destructive, non-adjacent block reducer. Visible-text replay + # identity is intentionally too weak here because reasoning, durable ids, + # attachments, and provider payload can distinguish legitimate repeated + # rows. Only token-tolerant canonical payload equality authorizes removal. + existing_keys = [_canonical_replay_digest(m) for m in existing_messages] + candidate_keys = [_canonical_replay_digest(m) for m in candidates] existing_large = [m for m in existing_messages if isinstance(m, dict)] cleaned = [] idx = 0 @@ -6965,7 +7841,10 @@ def _strip_replayed_context_items(existing_messages, candidates): while ( idx + length < len(candidate_keys) and start + length < len(existing_keys) - and candidate_keys[idx + length] == existing_keys[start + length] + and _comparison_keys_equal( + candidate_keys[idx + length], + existing_keys[start + length], + ) ): length += 1 if length > best: @@ -6979,24 +7858,54 @@ def _strip_replayed_context_items(existing_messages, candidates): return cleaned -def _dedupe_replayed_context_messages(previous_context, result_messages, msg_text=None): +def _dedupe_replayed_context_messages( + previous_context, + result_messages, + msg_text=None, + *, + result_prefix_is_authoritative=None, +): """Keep model context append-only without replayed blocks/summaries.""" previous_context = list(previous_context or []) result_messages = list(result_messages or []) if not previous_context or not result_messages: return result_messages previous_user_tail = _stale_user_tail_candidate(_last_user_row(previous_context)) - if not _messages_have_prefix( + has_authoritative_prefix = result_prefix_is_authoritative is True + if result_prefix_is_authoritative is None: + has_authoritative_prefix = _messages_have_prefix( + result_messages, + previous_context, + key_fn=_message_replay_key, + ) + if not has_authoritative_prefix and any( + type(message) is dict and message.get('role') == 'user' + for message in previous_context + ) and _messages_have_prefix( result_messages, previous_context, - key_fn=_message_replay_key, + key_fn=_canonical_replay_digest, ): + # A byte-exact canonical replay of the complete durable context — + # containing at least one real historical user turn — is conclusive + # history even when strict turn provenance was unavailable or was + # rejected upstream. The comparator is the payload-strict digest + # (tolerant only of the request-local ``_active_turn_token`` that + # ``_sanitize_messages_for_agent()`` strips), so payload-distinct + # lookalikes still fail closed. Reclassifying keeps the durable + # ``previous_context`` rows authoritative instead of letting the + # Agent's sanitized copy replace them wholesale, which would drop + # their turn-token bookkeeping and let a later dedup collapse the + # historical user row into the current turn. + has_authoritative_prefix = True + if not has_authoritative_prefix: # Agent-side role-sequence repair can replace the last prior user row # with a repaired current-user row. In that shape the result no longer # has `previous_context` as an exact prefix, but it should still be # merged as: previous context + clean current turn + assistant/tool delta. if ( - msg_text + result_prefix_is_authoritative is not False + and msg_text and len(previous_context) >= 1 and len(result_messages) >= len(previous_context) and _messages_have_prefix( @@ -7030,7 +7939,15 @@ def _dedupe_replayed_context_messages(previous_context, result_messages, msg_tex candidates = [cleaned_boundary] + result_messages[boundary_idx + 1:] else: candidates = result_messages[boundary_idx:] - candidates = _strip_replayed_prefix(previous_context, candidates) + # Payload-strict reduction on this destructive branch too: the + # persisted prefix rows carry durable ids/token bookkeeping the + # replayed copies lack, so only canonical payload identity + # (token-tolerant) may authorize dropping a candidate row. + candidates = _strip_replayed_prefix( + previous_context, + candidates, + key_fn=_canonical_replay_digest, + ) if candidates: candidates = _strip_replayed_context_items(previous_context, candidates) return previous_context + candidates @@ -7043,7 +7960,11 @@ def _dedupe_replayed_context_messages(previous_context, result_messages, msg_tex for m in result_messages ) if assistant_or_tool_only_result: - candidates = _strip_replayed_prefix(previous_context, result_messages) + candidates = _strip_replayed_prefix( + previous_context, + result_messages, + key_fn=_canonical_replay_digest, + ) if candidates: candidates = _strip_replayed_context_items(previous_context, candidates) return previous_context + candidates @@ -7059,7 +7980,18 @@ def _dedupe_replayed_context_messages(previous_context, result_messages, msg_tex previous_user_tail, previous_context=previous_context, ) - candidates = _strip_replayed_prefix(previous_context, candidates) + # Mirror the display projection's payload-strict reduction for the + # authoritative-prefix delta: only byte-identical canonical payloads are + # replay duplicates here, tolerating solely the request-local + # ``_active_turn_token`` that ``_sanitize_messages_for_agent()`` strips + # from the history the Agent replays back. The weak visible-text key + # ignores ``reasoning``/``id``/token and can misjudge payload-distinct + # rows on this destructive path. + candidates = _strip_replayed_prefix( + previous_context, + candidates, + key_fn=_canonical_replay_digest, + ) if candidates: candidates = _strip_replayed_context_items(previous_context, candidates) return previous_context + candidates @@ -7610,6 +8542,7 @@ def _merge_display_messages_after_agent_result( msg_text, source: str = "webui", verification_nudge_provenance=None, + result_has_authoritative_full_history_prefix=False, ): """Keep UI transcript durable while allowing model context to compact. @@ -7633,30 +8566,7 @@ def _merge_display_messages_after_agent_result( # three inputs consistently so prefix/delta detection below stays aligned. # (#5334; same internal-control-message class as #3320/#3821/#4373/#4875) previous_display = _drop_synthetic_control_messages(previous_display) - # Deduplicate stale _partial messages that accumulated in previous_display. - # A bug in cancel_stream() could insert multiple identical _partial messages - # when _stripped was empty but _has_reasoning/_has_tools was True. The - # merge's _message_identity previously returned None for empty _partial - # messages, so the seen-set couldn't catch them — they doubled each turn. - # Scan backwards and keep only the LAST occurrence of each unique _partial - # identity, then reverse back to original order. - _partial_seen = set() - _deduped_rev = [] - for m in reversed(previous_display): - if isinstance(m, dict) and m.get('_partial'): - key = _message_identity(m) - if key is not None: - if key in _partial_seen: - continue - _partial_seen.add(key) - _deduped_rev.append(m) - _deduped = list(reversed(_deduped_rev)) - if len(_deduped) < len(previous_display): - logger.debug( - "Deduplicated %d stale _partial messages from previous_display (was %d, now %d)", - len(previous_display) - len(_deduped), len(previous_display), len(_deduped), - ) - previous_display = _deduped + previous_display, _ = _collapse_replayed_assistant_rows(previous_display) previous_context = list(previous_context or []) result_messages = list(result_messages or []) if isinstance(verification_nudge_provenance, dict): @@ -7683,6 +8593,8 @@ def _merge_display_messages_after_agent_result( # would otherwise slip into the merged transcript as a real delta. (#5334) previous_context = _drop_synthetic_control_messages(previous_context) result_messages = _drop_synthetic_control_messages(result_messages) + previous_context, _ = _collapse_replayed_assistant_rows(previous_context) + result_messages, _ = _collapse_replayed_assistant_rows(result_messages) if not result_messages: return previous_display active_turn_row_index = _find_active_turn_checkpoint_index( @@ -7725,6 +8637,25 @@ def _merge_display_messages_after_agent_result( display_message['attachments'] = copy.deepcopy( active_turn_row['attachments'] ) + if not result_has_authoritative_full_history_prefix: + result_has_authoritative_full_history_prefix = ( + _result_has_authoritative_full_history_prefix( + result_messages, + previous_context, + _active_turn_identity, + msg_text, + ) + ) + if not result_has_authoritative_full_history_prefix: + result_messages, result_has_authoritative_full_history_prefix = ( + _reconcile_payload_distinct_history_prefix( + previous_context, + result_messages, + _active_turn_identity, + msg_text, + source, + ) + ) previous_user_tail = _stale_user_tail_candidate(_last_user_row(previous_context)) # ── Backfill normal turns from previous_context that are missing from @@ -7745,116 +8676,105 @@ def _merge_display_messages_after_agent_result( previous_context, ) ) - _display_id_set = {_message_identity(m) for m in previous_display} - _context_id_set = { - _message_identity(m) - for m in previous_context - if not ( - isinstance(m, dict) - and m.get('_active_turn_token') in _displayed_native_image_context_tokens - ) - if not _is_context_compression_marker(m) - and not _is_compressed_context_tool_result_summary_message(m) + _display_keys, context_keys = _display_backfill_projection_keys( + previous_display, + previous_context, + ) + # A trusted token pairs the provider-facing native-image row with its + # text-only display mirror even though their payloads are intentionally + # different. Give only those proven pairs one shared projection key so + # strict backfill never exposes the native image row as a second turn. + for _projection, _keys in ( + (previous_display, _display_keys), + (previous_context, context_keys), + ): + for _index, _message in enumerate(_projection): + _token = ( + _message.get('_active_turn_token') + if isinstance(_message, dict) + else None + ) + if _token in _displayed_native_image_context_tokens: + _keys[_index] = ('native_image_display_mirror', _token) + _display_counts = Counter(_display_keys) + _context_counts = Counter(context_keys) + # A count budget preserves duplicate multiplicity. Sets would hide a + # second context occurrence merely because one display occurrence has + # the same projection identity. + _insert_budget = { + key: count - _display_counts.get(key, 0) + for key, count in _context_counts.items() + if count > _display_counts.get(key, 0) } - _has_context_only_turns = bool(_context_id_set - _display_id_set) + _has_context_only_turns = bool(_insert_budget) if _has_context_only_turns: - context_keys = [ - None - if ( - isinstance(m, dict) - and m.get('_active_turn_token') in _displayed_native_image_context_tokens - ) - else _message_identity(m) - for m in previous_context - ] - # Precompute display keys once; avoids repeated json.dumps calls inside - # the inner any() loop (was O(D²·C) — see perf fix below). - _display_keys = [_message_identity(m) for m in previous_display] - # Multiset mirror of context_keys[_cursor:] kept in sync as _cursor - # advances. Enables O(1) membership tests in the any() check instead - # of an O(N) list scan, while preserving EXACT list-slice semantics: - # _message_identity intentionally returns duplicate keys for - # identical-content turns (and None for empty rows), so a plain set - # would drop a key still present later in the slice. A count-keyed - # dict (including None) matches `in context_keys[_cursor:]` exactly. - _remaining_ck_counts = {} - for _ck in context_keys: - _remaining_ck_counts[_ck] = _remaining_ck_counts.get(_ck, 0) + 1 + _positions_by_key = {} + for _index, _key in enumerate(context_keys): + _positions_by_key.setdefault(_key, []).append(_index) + _remaining_ck_counts = dict(_context_counts) + _future_display_counts = dict(_display_counts) + _shared_remaining_keys = { + key + for key in _remaining_ck_counts + if _future_display_counts.get(key, 0) > 0 + } _backfilled = [] - # #3300 fix: track ONLY context rows we splice in, so the - # visible-display backbone is never suppressed. Sharing one set - # between context inserts and display rows (and _message_identity - # ignoring timestamps) dropped a legitimate second identical visible - # user turn. Display rows are always appended in order; a context - # row is backfilled only if it isn't already a display row and - # hasn't already been inserted. - _context_inserted = set() _cursor = 0 + + def _backfill_context_range(start, stop): + for _context_idx in range(start, stop): + _ckey = context_keys[_context_idx] + _cmsg = previous_context[_context_idx] + if ( + _insert_budget.get(_ckey, 0) > 0 + and not _is_context_compression_marker(_cmsg) + and not _is_compressed_context_tool_result_summary_message(_cmsg) + ): + _backfilled.append(copy.deepcopy(_cmsg)) + _insert_budget[_ckey] -= 1 + + def _consume_context_range(start, stop): + for _context_idx in range(start, stop): + _consumed_key = context_keys[_context_idx] + _remaining = _remaining_ck_counts.get(_consumed_key, 0) - 1 + if _remaining <= 0: + _remaining_ck_counts.pop(_consumed_key, None) + _shared_remaining_keys.discard(_consumed_key) + else: + _remaining_ck_counts[_consumed_key] = _remaining + for _display_idx, _dmsg in enumerate(previous_display): _dkey = _display_keys[_display_idx] - if _dkey is not None: - _j = _cursor - while _j < len(context_keys) and context_keys[_j] != _dkey: - _j += 1 - if _j < len(context_keys): - for _k in range(_cursor, _j): - _ckey = context_keys[_k] - _cmsg = previous_context[_k] - if ( - _ckey is not None - and _ckey not in _context_inserted - and _ckey not in _display_id_set - and not _is_context_compression_marker(_cmsg) - and not _is_compressed_context_tool_result_summary_message(_cmsg) - ): - _backfilled.append(copy.deepcopy(_cmsg)) - _context_inserted.add(_ckey) - # Sync multiset: decrement keys consumed by advancing - # the cursor to _j+1 (delete at zero so membership matches - # the list slice exactly). - for _k in range(_cursor, _j + 1): - _consumed_ck = context_keys[_k] - _ck_n = _remaining_ck_counts.get(_consumed_ck, 0) - 1 - if _ck_n <= 0: - _remaining_ck_counts.pop(_consumed_ck, None) - else: - _remaining_ck_counts[_consumed_ck] = _ck_n - _cursor = _j + 1 - elif not any( - _display_keys[_fi] in _remaining_ck_counts - for _fi in range(_display_idx + 1, len(_display_keys)) - ): - for _k in range(_cursor, len(context_keys)): - _ckey = context_keys[_k] - _cmsg = previous_context[_k] - if ( - _ckey is not None - and _ckey not in _context_inserted - and _ckey not in _display_id_set - and not _is_context_compression_marker(_cmsg) - and not _is_compressed_context_tool_result_summary_message(_cmsg) - ): - _backfilled.append(copy.deepcopy(_cmsg)) - _context_inserted.add(_ckey) - _cursor = len(context_keys) - _remaining_ck_counts.clear() + _future_count = _future_display_counts.get(_dkey, 0) - 1 + if _future_count <= 0: + _future_display_counts.pop(_dkey, None) + _shared_remaining_keys.discard(_dkey) + else: + _future_display_counts[_dkey] = _future_count + + _positions = _positions_by_key.get(_dkey, ()) + _position_idx = bisect_left(_positions, _cursor) + _j = ( + _positions[_position_idx] + if _position_idx < len(_positions) + else len(context_keys) + ) + if _j < len(context_keys): + _backfill_context_range(_cursor, _j) + _consume_context_range(_cursor, _j + 1) + _cursor = _j + 1 + elif not _shared_remaining_keys: + # No later display row can anchor the remaining context. + # Preserve the historical ordering by splicing the tail + # before this unmatched visible row. + _backfill_context_range(_cursor, len(context_keys)) + _consume_context_range(_cursor, len(context_keys)) + _cursor = len(context_keys) # The display row is the visible backbone — always preserve it, # in order, even when an earlier (identical-content) turn or a # backfilled context row shares its timestamp-less identity. _backfilled.append(_dmsg) - while _cursor < len(context_keys): - _ckey = context_keys[_cursor] - _cmsg = previous_context[_cursor] - _cursor += 1 - if ( - _ckey is not None - and _ckey not in _context_inserted - and _ckey not in _display_id_set - and not _is_context_compression_marker(_cmsg) - and not _is_compressed_context_tool_result_summary_message(_cmsg) - ): - _backfilled.append(copy.deepcopy(_cmsg)) - _context_inserted.add(_ckey) + _backfill_context_range(_cursor, len(context_keys)) if len(_backfilled) > len(previous_display): logger.debug( "Backfilled %d context-only turns into previous_display (was %d, now %d)", @@ -7864,7 +8784,22 @@ def _merge_display_messages_after_agent_result( ) previous_display = _backfilled - if _messages_have_prefix(result_messages, previous_context): + result_prefix_is_authoritative = ( + result_has_authoritative_full_history_prefix is True + ) + if not result_prefix_is_authoritative and not _active_turn_identity: + # Legacy direct callers without provenance retain visible-prefix + # compatibility. Once strict turn provenance rejects a prefix, never + # let this weaker comparator reclassify and delete the same rows. + result_prefix_is_authoritative = _messages_have_prefix( + result_messages, + previous_context, + allow_exact_payload=False, + ) + if result_prefix_is_authoritative: + # Exact full-history authority is established before stable IDs are + # minted. Trust that frozen decision here instead of re-comparing the + # now-stamped result prefix against the original idless context. candidates = result_messages[len(previous_context):] # Normalize stale merges only in the new-turn slice; never rewrite # historical rows in the already-committed previous_context prefix. @@ -7889,10 +8824,48 @@ def _merge_display_messages_after_agent_result( for m in candidates ) if not (assistant_or_tool_only_candidates and not current_user_in_candidates): - candidates = _strip_replayed_prefix(previous_display, candidates) - candidates = _strip_replayed_prefix(previous_context, candidates) + candidates = _strip_replayed_prefix( + previous_display, + candidates, + key_fn=_canonical_message_digest, + ) + candidates = _strip_replayed_prefix( + previous_context, + candidates, + key_fn=_canonical_message_digest, + ) else: - current_user_idx = _find_current_user_turn(result_messages, msg_text) + if ( + isinstance(_active_turn_identity, dict) + and _active_turn_identity.get('token') + ): + # A pending WebUI token makes visible-text prompt matching + # insufficient: repeated prompts can point at a historical user and + # reclassify its assistant as current output. Accept only the token + # itself or the single Agent-authoritative index domain. Legacy + # callers without a token retain the old visible fallback below. + current_user_idx = _find_active_turn_checkpoint_index( + result_messages, + previous_context, + _active_turn_identity, + msg_text, + ) + if current_user_idx is None and not any( + isinstance(message, dict) + and message.get('role') == 'user' + and _normalize_user_text(_message_text(message.get('content'))) + == _normalize_user_text(msg_text) + for message in previous_context + ): + # Legacy Agents may not export the turn id/index pair. Visible + # matching remains safe only when the prompt is absent from the + # durable history; repeated prompts must fail closed above. + current_user_idx = _find_current_user_turn( + result_messages, + msg_text, + ) + else: + current_user_idx = _find_current_user_turn(result_messages, msg_text) assistant_or_tool_only_result = bool(result_messages) and all( _is_context_compression_marker(m) or ( @@ -8038,18 +9011,6 @@ def _merge_display_messages_after_agent_result( ): merged[-1]['id'] = msg['id'] continue - if ( - key is not None - and isinstance(msg, dict) - and msg.get('role') == 'assistant' - and merged - and _message_identity(merged[-1]) == key - ): - # Some provider/result replay paths can include the same assistant - # message twice in the current delta. Treat only adjacent identity - # matches as replay duplicates so identical answers in separate - # user turns remain visible. - continue if _is_context_compression_marker(msg) and key is not None and key in seen: continue display_msg = msg @@ -8066,6 +9027,7 @@ def _merge_display_messages_after_agent_result( merged.append(copy.deepcopy(display_msg)) if key is not None: seen.add(key) + merged, _ = _collapse_replayed_assistant_rows(merged) return merged @@ -8089,7 +9051,11 @@ def _assistant_reply_added_after_current_turn(result_messages, previous_context, """Return True only when the just-finished turn produced assistant text.""" result_messages = list(result_messages or []) previous_context = list(previous_context or []) - if _messages_have_prefix(result_messages, previous_context): + if _messages_have_prefix( + result_messages, + previous_context, + allow_exact_payload=True, + ): candidates = result_messages[len(previous_context):] else: current_user_idx = _find_current_user_turn(result_messages, msg_text) diff --git a/tests/test_context_message_dedup.py b/tests/test_context_message_dedup.py index 8a89a54d5c8..6cacb944a34 100644 --- a/tests/test_context_message_dedup.py +++ b/tests/test_context_message_dedup.py @@ -48,12 +48,9 @@ def test_deduplicate_context_messages_preserves_identical_answers_in_different_t ] result = _deduplicate_context_messages(messages) - # _message_identity is identity-based, not turn-aware: - # second assistant "4" has the same identity as first → removed. - # Second user "what is 3+1?" has different content → kept. - # This is intentional: the dedup catches context pollution from - # merge_session_messages_append_only, not replayed turns. - assert len(result) == 3 # user "2+2", assistant "4", user "3+1" + # Assistant replay reduction is adjacent and payload-strict. The distinct + # user boundary proves these are separate turns, so both answers remain. + assert len(result) == 4 def test_deduplicate_context_messages_empty_input(): @@ -63,7 +60,7 @@ def test_deduplicate_context_messages_empty_input(): assert _deduplicate_context_messages(None) is None -def test_deduplicate_context_messages_with_tool_calls(): +def test_deduplicate_context_messages_preserves_tool_call_rows(): from api.streaming import _deduplicate_context_messages messages = [ @@ -73,11 +70,13 @@ def test_deduplicate_context_messages_with_tool_calls(): ] result = _deduplicate_context_messages(messages) - assert len(result) == 2 # third message (dup) removed + # Structured provider payloads are fail-closed: even apparently identical + # rows stay durable unless the dedicated replay classifier proves them safe. + assert len(result) == 3 -def test_deduplicate_context_messages_different_timestamps_same_content(): - """Messages with same content but different timestamps should be deduped.""" +def test_deduplicate_context_messages_preserves_timestamp_distinct_assistants(): + """Provider-facing assistant payloads with distinct timestamps remain.""" from api.streaming import _deduplicate_context_messages messages = [ @@ -88,7 +87,12 @@ def test_deduplicate_context_messages_different_timestamps_same_content(): ] result = _deduplicate_context_messages(messages) - assert len(result) == 2 # duplicates removed despite different timestamps + assert len(result) == 3 + assert [message["role"] for message in result] == [ + "user", + "assistant", + "assistant", + ] def test_message_identity_strips_workspace_prefix(): diff --git a/tests/test_issue2592_partial_dedupe.py b/tests/test_issue2592_partial_dedupe.py index 789633c4bba..04a8e59d09d 100644 --- a/tests/test_issue2592_partial_dedupe.py +++ b/tests/test_issue2592_partial_dedupe.py @@ -1,4 +1,20 @@ +import copy import json +import threading + +import pytest + + +def _incomplete_reasoning_only(message_id, *, reasoning="encrypted reasoning", timestamp=123): + return { + "id": message_id, + "role": "assistant", + "content": "", + "timestamp": timestamp, + "finish_reason": "incomplete", + "reasoning": reasoning, + "codex_reasoning_items": [{"type": "reasoning", "encrypted_content": "opaque"}], + } def _tool_partial(reasoning="same reasoning", args=None, *, timestamp=123): @@ -85,3 +101,539 @@ def test_session_load_collapses_adjacent_duplicate_partials(tmp_path, monkeypatc assert sum(1 for message in persisted["messages"] if message.get("_partial")) == 1 assert persisted["updated_at"] == 200.0 assert (session_dir / f"{sid}.json.bak").exists() + + +def test_reasoning_only_incomplete_identity_requires_exact_payload(): + from api.streaming import _message_identity + + first = _incomplete_reasoning_only(1701) + replay = dict(first) + same_id_distinct_payload = _incomplete_reasoning_only(1701, timestamp=999) + distinct = _incomplete_reasoning_only(1702) + + assert _message_identity(first) == _message_identity(replay) + assert _message_identity(first) != _message_identity(same_id_distinct_payload) + assert _message_identity(first) != _message_identity(distinct) + assert _message_identity({"role": "assistant", "content": "", "finish_reason": "incomplete"}) is None + + +def test_session_load_collapses_non_adjacent_duplicate_incomplete_ids(tmp_path, monkeypatch): + import api.models as models + + sid = "fd05-copy" + session_dir = tmp_path / "sessions" + session_dir.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", session_dir / "_index.json") + first = _incomplete_reasoning_only(1701, reasoning="first") + second = _incomplete_reasoning_only(1702, reasoning="second") + payload = { + "session_id": sid, + "title": "FD05 duplicated incomplete responses", + "workspace": str(tmp_path), + "model": "gpt-5.6", + "created_at": 100.0, + "updated_at": 200.0, + "messages": [ + {"role": "user", "content": "run this"}, + first, + second, + dict(first), + dict(second), + dict(first), + dict(second), + {"role": "assistant", "content": "final visible answer"}, + ], + "tool_calls": [], + } + (session_dir / f"{sid}.json").write_text(json.dumps(payload), encoding="utf-8") + + loaded = models.Session.load(sid) + + assert loaded is not None + assert [message.get("id") for message in loaded.messages if message.get("id")] == [1701, 1702] + persisted = json.loads((session_dir / f"{sid}.json").read_text(encoding="utf-8")) + assert [message.get("id") for message in persisted["messages"] if message.get("id")] == [1701, 1702] + assert persisted["updated_at"] == 200.0 + assert (session_dir / f"{sid}.json.bak").exists() + + +def test_context_dedupe_is_idempotent_for_alternating_incomplete_ids(): + from api.streaming import _deduplicate_context_messages + + first = _incomplete_reasoning_only(1701, reasoning="first") + second = _incomplete_reasoning_only(1702, reasoning="second") + messages = [first, second, dict(first), dict(second)] * 10 + + once = _deduplicate_context_messages(messages) + twice = _deduplicate_context_messages(once) + + assert [message["id"] for message in once] == [1701, 1702] + assert twice == once + + +def test_display_merge_dedupes_incomplete_ids_after_state_db_reconciliation(): + from api.streaming import _merge_display_messages_after_agent_result + + first = _incomplete_reasoning_only(1701, reasoning="first") + second = _incomplete_reasoning_only(1702, reasoning="second") + user = {"role": "user", "content": "next", "id": 1703} + answer = {"role": "assistant", "content": "done", "id": 1704, "finish_reason": "stop"} + + merged = _merge_display_messages_after_agent_result( + [first, second, dict(first), dict(second)], + [first, second], + [first, second, dict(first), dict(second), user, answer], + "next", + ) + + ids = [message.get("id") for message in merged] + assert ids.count(1701) == 1 + assert ids.count(1702) == 1 + assert ids[-2:] == [1703, 1704] + + +def test_save_is_a_final_idempotent_barrier_for_incomplete_message_ids(tmp_path, monkeypatch): + from api import models + + session_dir = tmp_path / "sessions" + session_dir.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", tmp_path / "index.json") + + first = _incomplete_reasoning_only(1701, reasoning="first") + second = _incomplete_reasoning_only(1702, reasoning="second") + session = models.Session( + session_id="save-barrier", + messages=[first, second, dict(first), dict(second)], + ) + + session.save(skip_index=True) + session.save(skip_index=True) + + # save() never rebinds or mutates the list visible to active workers. + assert [message["id"] for message in session.messages] == [1701, 1702, 1701, 1702] + persisted = json.loads((session_dir / "save-barrier.json").read_text(encoding="utf-8")) + assert [message["id"] for message in persisted["messages"]] == [1701, 1702] + assert persisted["message_count"] == 2 + + +def test_save_snapshot_does_not_lose_concurrent_alias_append(tmp_path, monkeypatch): + from api import models + + session_dir = tmp_path / "sessions" + session_dir.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", tmp_path / "index.json") + + first = _incomplete_reasoning_only(1701, reasoning="first") + session = models.Session(session_id="concurrent-alias", messages=[first, dict(first)]) + live_alias = session.messages + concurrent = {"role": "user", "content": "arrived during save", "id": 1702} + real_collapse = models._collapse_duplicate_incomplete_message_ids + + def collapse_then_append(messages): + collapsed = real_collapse(messages) + live_alias.append(concurrent) + return collapsed + + monkeypatch.setattr(models, "_collapse_duplicate_incomplete_message_ids", collapse_then_append) + session.save(skip_index=True) + + assert session.messages is live_alias + assert session.messages[-1] == concurrent + first_payload = json.loads((session_dir / "concurrent-alias.json").read_text(encoding="utf-8")) + assert [message["id"] for message in first_payload["messages"]] == [1701] + + monkeypatch.setattr(models, "_collapse_duplicate_incomplete_message_ids", real_collapse) + session.save(skip_index=True) + second_payload = json.loads((session_dir / "concurrent-alias.json").read_text(encoding="utf-8")) + assert [message["id"] for message in second_payload["messages"]] == [1701, 1702] + + +def test_recovery_does_not_resurrect_duplicate_incomplete_backup(tmp_path): + from api.session_recovery import inspect_session_recovery_status, recover_session + + session_path = tmp_path / "repaired.json" + backup_path = tmp_path / "repaired.json.bak" + first = _incomplete_reasoning_only(1701, reasoning="first") + second = _incomplete_reasoning_only(1702, reasoning="second") + session_path.write_text(json.dumps({"messages": [first, second]}), encoding="utf-8") + backup_path.write_text( + json.dumps({"messages": [first, second, dict(first), dict(second)]}), + encoding="utf-8", + ) + + status = inspect_session_recovery_status(session_path) + assert status["live_messages"] == 2 + assert status["bak_messages"] == 2 + assert status["recommend"] == "no_action" + assert recover_session(session_path)["restored"] is False + + +def test_recovery_still_restores_unique_backup_excess(tmp_path): + from api.session_recovery import inspect_session_recovery_status + + session_path = tmp_path / "repaired.json" + backup_path = tmp_path / "repaired.json.bak" + first = _incomplete_reasoning_only(1701, reasoning="first") + unique = {"role": "user", "content": "must survive", "id": 1702} + session_path.write_text(json.dumps({"messages": [first]}), encoding="utf-8") + backup_path.write_text(json.dumps({"messages": [first, unique]}), encoding="utf-8") + + status = inspect_session_recovery_status(session_path) + assert status["live_messages"] == 1 + assert status["bak_messages"] == 2 + assert status["recommend"] == "restore" + + +def test_typed_incomplete_ids_round_trip_without_cross_type_collision(): + from api.models import ( + _collapse_duplicate_incomplete_message_ids, + _strict_incomplete_message_id_key, + ) + + # Each admissible scalar type carries its own deletion authority. + assert _strict_incomplete_message_id_key("abc") == ("str", "abc") + assert _strict_incomplete_message_id_key(1701) == ("int", 1701) + assert _strict_incomplete_message_id_key(1.5) == ("float", 1.5) + + # 1 vs "1" vs 1.0 vs True vs "True" vs b"1" are DISTINCT rows: none may + # collapse into another's bucket, and the bytes id must round-trip intact. + typed_ids = [1, "1", 1.0, True, "True", b"1"] + rows = [_incomplete_reasoning_only(message_id) for message_id in typed_ids] + collapsed, _ = _collapse_duplicate_incomplete_message_ids(rows) + assert len(collapsed) == len(rows) + kept = [message["id"] for message in collapsed] + assert sum(1 for k in kept if type(k) is int and k == 1) == 1 + assert sum(1 for k in kept if type(k) is str and k == "1") == 1 + assert sum(1 for k in kept if type(k) is float and k == 1.0) == 1 + assert sum(1 for k in kept if type(k) is str and k == "True") == 1 + assert sum(1 for k in kept if type(k) is bool and k is True) == 1 + assert sum(1 for k in kept if type(k) is bytes and k == b"1") == 1 + + # Same-type replays of an admissible id still collapse exactly as before. + for message_id in (1, "1", 1.5): + pair = [_incomplete_reasoning_only(message_id), _incomplete_reasoning_only(message_id)] + deduped, changed = _collapse_duplicate_incomplete_message_ids(pair) + assert changed is True + assert len(deduped) == 1 + assert deduped[0]["id"] == message_id + assert type(deduped[0]["id"]) is type(message_id) + + +def test_incomplete_id_rejects_bool_container_subclass_and_non_finite(): + from api.models import _strict_incomplete_message_id_key as key + + class StrId(str): + pass + + class IntId(int): + pass + + assert key(True) is None + assert key(False) is None + assert key(None) is None + assert key("") is None + assert key(["1"]) is None + assert key({"id": 1}) is None + assert key(("1",)) is None + assert key(b"1") is None + assert key(StrId("1")) is None + assert key(IntId(1)) is None + assert key(float("nan")) is None + assert key(float("inf")) is None + assert key(float("-inf")) is None + + +def test_mixed_type_backup_rows_remain_independently_recoverable(tmp_path): + from api.session_recovery import inspect_session_recovery_status + + session_path = tmp_path / "mixed.json" + backup_path = tmp_path / "mixed.json.bak" + int_row = _incomplete_reasoning_only(1) + str_row = _incomplete_reasoning_only("1") + session_path.write_text(json.dumps({"messages": [int_row]}), encoding="utf-8") + backup_path.write_text( + json.dumps({"messages": [int_row, str_row]}), encoding="utf-8" + ) + + status = inspect_session_recovery_status(session_path) + assert status["live_messages"] == 1 + # "1" is NOT a duplicate-only replay of 1: the distinct backup row keeps + # its recovery authority instead of being classified as a replay of 1. + assert status["bak_messages"] == 2 + assert status["recommend"] == "restore" + + +def test_save_deep_isolates_retained_rows_from_concurrent_mutation(tmp_path, monkeypatch): + from api import models + + session_dir = tmp_path / "sessions" + session_dir.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", tmp_path / "index.json") + + session = models.Session( + session_id="deep-isolation", + messages=[_incomplete_reasoning_only(1701)], + ) + live_row = session.messages[0] + real_deepcopy = copy.deepcopy + + def deepcopy_then_mutate(obj, *args, **kwargs): + snapshot = real_deepcopy(obj, *args, **kwargs) + if isinstance(obj, list) and obj and obj[0] is live_row: + # A worker mutates the LIVE nested dict in the window between the + # collapse scan and json.dumps; the persisted payload must not + # observe it. + live_row["codex_reasoning_items"][0]["encrypted_content"] = "MUTATED" + return snapshot + + monkeypatch.setattr(models.copy, "deepcopy", deepcopy_then_mutate) + session.save(skip_index=True) + + persisted = json.loads((session_dir / "deep-isolation.json").read_text(encoding="utf-8")) + assert persisted["messages"][0]["codex_reasoning_items"][0]["encrypted_content"] == "opaque" + assert session.messages[0]["codex_reasoning_items"][0]["encrypted_content"] == "MUTATED" + + +def test_save_owns_snapshot_before_duplicate_selection(tmp_path, monkeypatch): + from api import models + + session_dir = tmp_path / "sessions" + session_dir.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", tmp_path / "index.json") + + first = _incomplete_reasoning_only(1701, reasoning="first") + duplicate = copy.deepcopy(first) + session = models.Session(session_id="selection-isolation", messages=[first, duplicate]) + real_collapse = models._collapse_duplicate_incomplete_message_ids + selection_done = threading.Event() + mutation_done = threading.Event() + + def mutate_after_selection(): + assert selection_done.wait(timeout=2) + first["codex_reasoning_items"][0]["encrypted_content"] = "MUTATED" + mutation_done.set() + + worker = threading.Thread(target=mutate_after_selection) + worker.start() + + def collapse_then_mutate(messages): + selected = real_collapse(messages) + # Pause save after selection while a scheduled writer mutates the live + # row. This is exactly before the old post-selection deepcopy. + selection_done.set() + assert mutation_done.wait(timeout=2) + return selected + + monkeypatch.setattr(models, "_collapse_duplicate_incomplete_message_ids", collapse_then_mutate) + session.save(skip_index=True) + worker.join(timeout=2) + assert not worker.is_alive() + + persisted = json.loads((session_dir / "selection-isolation.json").read_text(encoding="utf-8")) + assert len(persisted["messages"]) == 1 + assert persisted["messages"][0]["codex_reasoning_items"][0]["encrypted_content"] == "opaque" + + +def test_save_writes_index_from_same_collapsed_snapshot(tmp_path, monkeypatch): + from api import models + + session_dir = tmp_path / "sessions" + session_dir.mkdir() + index_file = tmp_path / "index.json" + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", index_file) + + session = models.Session( + session_id="index-parity", + messages=[_incomplete_reasoning_only(1701, timestamp=100)], + ) + session.save() # full-rebuild path creates the index + + # An exact replayed duplicate lands in the live list. + session.messages.append(dict(session.messages[0])) + session._metadata_message_count = 2 + session.save() # fast path: _write_session_index(updates=[self]) + + sidecar = json.loads((session_dir / "index-parity.json").read_text(encoding="utf-8")) + assert sidecar["message_count"] == 1 + assert len(sidecar["messages"]) == 1 + + index_entries = json.loads(index_file.read_text(encoding="utf-8")) + entry = next(e for e in index_entries if e["session_id"] == "index-parity") + # The sidebar index must reflect the SAME collapsed snapshot. + assert entry["message_count"] == sidecar["message_count"] == 1 + assert entry["last_message_at"] == 100 + + +@pytest.mark.parametrize("first_generation", ["one", "two"]) +def test_same_sid_saves_publish_one_complete_generation_in_both_orders( + tmp_path, monkeypatch, first_generation +): + from api import models + + session_dir = tmp_path / "sessions" + session_dir.mkdir() + index_file = session_dir / "_index.json" + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", index_file) + + one = models.Session( + session_id="shared-save-authority", + title="generation-one", + model="model-one", + messages=[{"role": "user", "content": "one", "timestamp": 100}], + ) + two = models.Session( + session_id="shared-save-authority", + title="generation-two", + model="model-two", + messages=[ + {"role": "user", "content": "one", "timestamp": 100}, + {"role": "assistant", "content": "two", "timestamp": 200}, + ], + ) + generations = {"one": one, "two": two} + second_generation = "two" if first_generation == "one" else "one" + first_reached_index = threading.Event() + release_first = threading.Event() + second_started = threading.Event() + real_write_index = models._write_session_index + first_thread_id = {"value": None} + + def gated_write_index(*args, **kwargs): + if threading.get_ident() == first_thread_id["value"]: + first_reached_index.set() + assert release_first.wait(timeout=2) + return real_write_index(*args, **kwargs) + + monkeypatch.setattr(models, "_write_session_index", gated_write_index) + + def save_first(): + first_thread_id["value"] = threading.get_ident() + generations[first_generation].save(touch_updated_at=False) + + def save_second(): + second_started.set() + generations[second_generation].save(touch_updated_at=False) + + first = threading.Thread(target=save_first) + second = threading.Thread(target=save_second) + first.start() + assert first_reached_index.wait(timeout=2) + second.start() + assert second_started.wait(timeout=2) + release_first.set() + first.join(timeout=2) + second.join(timeout=2) + assert not first.is_alive() + assert not second.is_alive() + + sidecar = json.loads((session_dir / "shared-save-authority.json").read_text(encoding="utf-8")) + index = json.loads(index_file.read_text(encoding="utf-8")) + row = next(entry for entry in index if entry["session_id"] == "shared-save-authority") + expected = generations[second_generation] + assert sidecar["title"] == row["title"] == expected.title + assert sidecar["model"] == row["model"] == expected.model + assert sidecar["message_count"] == row["message_count"] == len(expected.messages) + assert row["user_message_count"] == expected._compute_user_message_count(expected.messages) + assert row["last_message_at"] == expected.messages[-1]["timestamp"] + + +def test_recovery_restores_the_collapsed_backup_payload(tmp_path): + from api.session_recovery import inspect_session_recovery_status, recover_session + + session_path = tmp_path / "restored.json" + backup_path = tmp_path / "restored.json.bak" + first = _incomplete_reasoning_only(1701, reasoning="first") + unique = {"role": "user", "content": "must survive", "id": 1702} + session_path.write_text(json.dumps({"messages": [first]}), encoding="utf-8") + backup_path.write_text( + json.dumps({"messages": [first, dict(first), unique], "message_count": 3}), + encoding="utf-8", + ) + + status = inspect_session_recovery_status(session_path) + assert status["recommend"] == "restore" + result = recover_session(session_path) + assert result["restored"] is True + + # The restore writes the SAME effective payload _msg_count() evaluated: + # the duplicate-only replay is NOT resurrected, the unique row survives, + # and message_count is recomputed from the collapsed payload. + restored = json.loads(session_path.read_text(encoding="utf-8")) + assert restored["messages"] == [first, unique] + assert restored["message_count"] == 2 + + +def test_reconciliation_and_persistence_share_incomplete_eligibility(): + from api.models import _incomplete_reasoning_message_id + from api.streaming import _message_identity + + def reconciliation_incomplete_key(message): + identity = _message_identity(message) + if ( + isinstance(identity, tuple) + and len(identity) == 4 + and str(identity[3]).startswith("__incomplete_message_id__") + ): + return identity[3] + return None + + base = _incomplete_reasoning_only(1701) + structured_blank = { + **base, + "content": [{"type": "output_text", "text": " hidden "}], + } + structured_visible = { + **base, + "content": [{"type": "output_text", "text": "visible answer"}], + } + cases_eligible = [ + base, + structured_blank, + {**base, "id": 1}, + {**base, "id": "1"}, + {**base, "id": 1.5}, + ] + cases_ineligible = [ + structured_visible, + {**base, "tool_call_id": "call_1"}, + {**base, "tool_calls": [{"id": "call_1"}]}, + {**base, "id": True}, + {**base, "id": ""}, + {**base, "id": None}, + {**base, "id": ["1"]}, + {**base, "id": float("nan")}, + ] + + partial_same_id_different_reasoning = [ + {**base, "_partial": True, "reasoning": "first"}, + {**base, "_partial": True, "reasoning": "second"}, + ] + partial_different_ids_same_reasoning = [ + {**base, "_partial": True, "id": 1701, "reasoning": "same"}, + {**base, "_partial": True, "id": 1702, "reasoning": "same"}, + ] + + # Blank content (plain or structured), tool-call identity, and malformed + # ids are classified identically by both layers: neither may drop a row + # the other considers distinct. + for message in cases_eligible: + assert _incomplete_reasoning_message_id(message) is not None + assert reconciliation_incomplete_key(message) is not None + for message in cases_ineligible: + assert _incomplete_reasoning_message_id(message) is None + assert reconciliation_incomplete_key(message) is None + assert ( + reconciliation_incomplete_key(partial_same_id_different_reasoning[0]) + != reconciliation_incomplete_key(partial_same_id_different_reasoning[1]) + ) + assert ( + reconciliation_incomplete_key(partial_different_ids_same_reasoning[0]) + != reconciliation_incomplete_key(partial_different_ids_same_reasoning[1]) + ) diff --git a/tests/test_issue7032_context_replay_token_identity.py b/tests/test_issue7032_context_replay_token_identity.py new file mode 100644 index 00000000000..92d5c9b3d40 --- /dev/null +++ b/tests/test_issue7032_context_replay_token_identity.py @@ -0,0 +1,190 @@ +"""Gate-remediation regression tests for nesquena/hermes-webui#7032. + +The 2026-08-16 gate certification reproduced one root cause with two +manifestations on head 661a8010: the display projection became +payload-strict while the model-context replay dedup path kept comparing +rows whose persisted copies carry the request-local ``_active_turn_token`` +(stripped from the history the Agent replays back by +``_sanitize_messages_for_agent``). A normal repeated-prompt turn then +either + +(a) duplicates the historical user row in model context, or +(b) drops the current exchange from model context (display/context + divergence). + +Both scenarios below run the real streaming caller sequence +(``_settle_result_messages``) against a turn-1 persisted state whose user +row carries ``_active_turn_token`` exactly as production persists it. +""" + +import copy +from types import SimpleNamespace + +PROMPT = "run the report again" + + +def _persisted_turn_one(*, with_api_content): + """Turn-1 state as the streaming writeback persists it.""" + user = { + "role": "user", + "content": PROMPT, + "id": 1, + "timestamp": 100.0, + "_active_turn_token": "direct-stream:100", + } + if with_api_content: + user["api_content"] = "[Workspace] " + PROMPT + assistant = { + "role": "assistant", + "content": "the report says A", + "id": 2, + "timestamp": 101.0, + } + return [user, assistant] + + +def _settle_repeated_prompt_turn(previous_context, result_messages, *, authoritative): + from api.streaming import _settle_result_messages + + previous_display = copy.deepcopy(previous_context) + session = SimpleNamespace( + session_id="gate-7032-regression", + messages=copy.deepcopy(previous_display), + context_messages=copy.deepcopy(previous_context), + truncation_watermark=None, + ) + identity = { + "token": "direct-stream:200", + "text": PROMPT, + "timestamp": 200.0, + "source": "webui", + "attachments": [], + "current_turn_user_idx": len(previous_context) if authoritative else None, + "turn_id": "turn:2" if authoritative else "", + } + _settle_result_messages( + session, + copy.deepcopy(previous_display), + copy.deepcopy(previous_context), + result_messages, + PROMPT, + "webui", + identity, + ) + return session + + +def _user_rows(messages): + return [ + message + for message in messages + if isinstance(message, dict) and message.get("role") == "user" + ] + + +def _assistant_answers(messages): + return [ + message.get("content") + for message in messages + if isinstance(message, dict) and message.get("role") == "assistant" + ] + + +def test_replayed_history_with_persisted_turn_token_is_not_duplicated(): + """(a) The historical user row must not be duplicated in model context. + + The Agent replays the sanitized history (no ``_active_turn_token``), + echoes the repeated current prompt, and answers. The persisted user + row and its replayed copy are the same durable turn; treating them as + payload-distinct reinserts history into the model context. + """ + from api.streaming import _sanitize_messages_for_agent + + previous_context = _persisted_turn_one(with_api_content=True) + replayed_history = _sanitize_messages_for_agent(previous_context) + result_messages = copy.deepcopy(replayed_history) + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": "the report now says B"}, + ] + + session = _settle_repeated_prompt_turn( + previous_context, + result_messages, + authoritative=True, + ) + + context_users = _user_rows(session.context_messages) + assert len(context_users) == 2, ( + f"historical user row duplicated in model context: {session.context_messages}" + ) + assert _assistant_answers(session.context_messages) == [ + "the report says A", + "the report now says B", + ] + assert len(_user_rows(session.messages)) == 2 + assert _assistant_answers(session.messages) == [ + "the report says A", + "the report now says B", + ] + + +def test_replayed_history_with_persisted_turn_token_legacy_authority(): + """(a) Same protection when an older Agent omits turn authority.""" + from api.streaming import _sanitize_messages_for_agent + + previous_context = _persisted_turn_one(with_api_content=True) + replayed_history = _sanitize_messages_for_agent(previous_context) + result_messages = copy.deepcopy(replayed_history) + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": "the report now says B"}, + ] + + session = _settle_repeated_prompt_turn( + previous_context, + result_messages, + authoritative=False, + ) + + context_users = _user_rows(session.context_messages) + assert len(context_users) == 2, ( + f"historical user row duplicated in model context: {session.context_messages}" + ) + assert _assistant_answers(session.context_messages) == [ + "the report says A", + "the report now says B", + ] + + +def test_current_answer_survives_full_history_replay_without_user_echo(): + """(b) The current answer must reach model context exactly once. + + The Agent replays the sanitized history and appends only the new + assistant answer (the current user turn is supplied out of band). + Model context and display must agree on the assistant answers; the + current answer must not be dropped and the historical answer must not + be duplicated. + """ + from api.streaming import _sanitize_messages_for_agent + + previous_context = _persisted_turn_one(with_api_content=False) + replayed_history = _sanitize_messages_for_agent(previous_context) + result_messages = copy.deepcopy(replayed_history) + [ + {"role": "assistant", "content": "the report now says B"}, + ] + + session = _settle_repeated_prompt_turn( + previous_context, + result_messages, + authoritative=False, + ) + + context_answers = _assistant_answers(session.context_messages) + display_answers = _assistant_answers(session.messages) + assert "the report now says B" in context_answers, ( + f"current answer dropped from model context: {session.context_messages}" + ) + assert context_answers == ["the report says A", "the report now says B"] + assert display_answers == context_answers, ( + "display/context divergence: " + f"display={display_answers} context={context_answers}" + ) diff --git a/tests/test_issue765_streaming_persistence.py b/tests/test_issue765_streaming_persistence.py index c3689b6976c..27d250758a1 100644 --- a/tests/test_issue765_streaming_persistence.py +++ b/tests/test_issue765_streaming_persistence.py @@ -277,28 +277,36 @@ class TestIssue765FollowupHardening: an exception fires before the checkpoint thread is created. """ - def test_same_session_concurrent_saves_use_distinct_temp_files(self, monkeypatch): - """Two concurrent saves of the same session must not collide on one tmp path. + def test_same_session_concurrent_saves_are_serialized_with_distinct_temp_files(self, monkeypatch): + """Concurrent saves must publish complete, independent generations. - The key regression guard here is that each save call should reach os.replace() - with a distinct source tmp path. With the old shared `.tmp` scheme, both - threads would target the same path and the second replace would deterministically - fail once the first consume/remove happened. + Each save still owns a distinct temporary file, while the per-session + authority serializes replacement of the durable sidecar. """ s = _make_session("same_sid") s.save(skip_index=True) # seed the file on disk original_replace = models.os.replace - barrier = threading.Barrier(2) replace_sources = [] + replace_active = 0 + max_replace_active = 0 + replace_state_lock = threading.Lock() errors = [] - def _replace_with_barrier(src, dst): - replace_sources.append(str(src)) - barrier.wait(timeout=5) - return original_replace(src, dst) + def _tracked_replace(src, dst): + nonlocal replace_active, max_replace_active + with replace_state_lock: + replace_sources.append(str(src)) + replace_active += 1 + max_replace_active = max(max_replace_active, replace_active) + try: + time.sleep(0.05) + return original_replace(src, dst) + finally: + with replace_state_lock: + replace_active -= 1 - monkeypatch.setattr(models.os, "replace", _replace_with_barrier) + monkeypatch.setattr(models.os, "replace", _tracked_replace) def _save_worker(): try: @@ -319,6 +327,7 @@ def _save_worker(): "Concurrent same-session saves must use distinct temp files even if Windows-safe " f"replace retries one of them; got {replace_sources}" ) + assert max_replace_active == 1, "Same-session sidecar publication must be serialized" data = json.loads(s.path.read_text(encoding="utf-8")) assert data["session_id"] == "same_sid" diff --git a/tests/test_merge_backfill_perf_optimization.py b/tests/test_merge_backfill_perf_optimization.py index 4a21e42469b..c53ae2e8d46 100644 --- a/tests/test_merge_backfill_perf_optimization.py +++ b/tests/test_merge_backfill_perf_optimization.py @@ -3,9 +3,9 @@ The optimization replaces an `in context_keys[_cursor:]` list-slice membership test with an O(1) count-keyed dict mirror. The subtle correctness requirement: -_message_identity intentionally returns DUPLICATE keys for identical-content -turns (and None for empty rows). A plain set would diverge from the original -list-slice semantics; a multiset (count dict, including None) is exact. +The projection-aware backfill key intentionally returns DUPLICATE keys for +exact rows and visible-identical non-assistant turns. A plain set would diverge +from the original list-slice semantics; a multiset count is exact. This test asserts the optimized merge produces byte-identical output to a reference implementation of the ORIGINAL list-slice semantics, over adversarial @@ -28,7 +28,7 @@ def _msg(role, text, **extra): def test_backfill_optimization_preserves_duplicate_identity_turns(): """Two identical-content user turns both survive the backfill merge. - _message_identity collapses identical user text to the same key. The + The backfill key collapses identical user text to the same key. The optimization must not drop the second identical turn (a plain-set mirror would). previous_display is the visible backbone; both 'Ok' user bubbles plus the interleaved assistant rows must be preserved in order. @@ -67,6 +67,136 @@ def test_backfill_optimization_preserves_duplicate_identity_turns(): assert contents.index("first reply") < contents.index("second reply") +def test_backfill_matches_stable_row_across_display_only_metadata(): + """Display enrichment must not make one stable assistant row appear twice.""" + previous_display = [ + _msg("user", "old prompt", id="old-user"), + _msg( + "assistant", + "first answer", + id="assistant-a", + _media_snapshots={"/tmp/result.png": "sha256:a"}, + _turnUsage={"input_tokens": 3, "output_tokens": 5}, + ), + ] + previous_context = [ + _msg("user", "old prompt", id="old-user"), + _msg("assistant", "first answer", id="assistant-a"), + _msg("assistant", "context-only answer", id="assistant-b"), + ] + result_messages = [ + *previous_context, + _msg("user", "current prompt", id="current-user"), + _msg("assistant", "final answer", id="assistant-final"), + ] + + merged = streaming._merge_display_messages_after_agent_result( + previous_display, + previous_context, + result_messages, + "current prompt", + result_has_authoritative_full_history_prefix=True, + ) + + assert [message.get("id") for message in merged] == [ + "old-user", + "assistant-a", + "assistant-b", + "current-user", + "assistant-final", + ] + assert merged[1]["_media_snapshots"] == {"/tmp/result.png": "sha256:a"} + assert merged[1]["_turnUsage"] == {"input_tokens": 3, "output_tokens": 5} + + +def test_backfill_projection_keys_keep_strict_ids_and_unknown_metadata(): + display = [ + _msg("assistant", "same", id=7, _statusCard={"phase": "done"}), + _msg("assistant", "typed", id=7), + _msg("assistant", "unknown", provider_metadata={"attempt": "alpha"}), + _msg("assistant", "one-sided"), + ] + context = [ + _msg("assistant", "same", id=7), + _msg("assistant", "typed", id="7"), + _msg("assistant", "unknown", provider_metadata={"attempt": "beta"}), + _msg("assistant", "one-sided", id=9), + ] + + display_keys, context_keys = streaming._display_backfill_projection_keys( + display, + context, + ) + + assert display_keys[0] == context_keys[0] + assert display_keys[1] != context_keys[1] + assert display_keys[2] != context_keys[2] + assert display_keys[3] == context_keys[3] + + +def test_backfill_preserves_duplicate_context_multiplicity(): + repeated = _msg("user", "repeat") + previous_display = [dict(repeated)] + previous_context = [dict(repeated), dict(repeated)] + + merged = streaming._merge_display_messages_after_agent_result( + previous_display, + previous_context, + [*previous_context, _msg("assistant", "final", id="assistant-final")], + "current prompt", + result_has_authoritative_full_history_prefix=True, + ) + + assert sum( + message.get("role") == "user" and message.get("content") == "repeat" + for message in merged + ) == 2 + + +def test_backfill_suffix_lookup_has_bounded_key_comparisons(monkeypatch): + comparisons = 0 + + class _Key: + def __init__(self, value): + self.value = value + + def __hash__(self): + return hash(self.value) + + def __eq__(self, other): + nonlocal comparisons + comparisons += 1 + return isinstance(other, _Key) and self.value == other.value + + size = 500 + anchor = _Key("anchor") + context_only = _Key("context-only") + display_keys = [_Key(f"display-{index}") for index in range(size)] + [anchor] + context_keys = [context_only, anchor] + monkeypatch.setattr( + streaming, + "_display_backfill_projection_keys", + lambda _display, _context: (display_keys, context_keys), + ) + previous_display = [ + _msg("user", f"display-{index}") + for index in range(size + 1) + ] + previous_context = [ + _msg("user", "context-only"), + _msg("user", "anchor"), + ] + + streaming._merge_display_messages_after_agent_result( + previous_display, + previous_context, + [_msg("assistant", "final")], + "current prompt", + ) + + assert comparisons < size * 20 + + def test_backfill_optimization_matches_reference_listslice_semantics(): """Differential check: optimized merge == reference (original) semantics over adversarial inputs with duplicate identities and empty rows.""" @@ -75,71 +205,75 @@ def test_backfill_optimization_matches_reference_listslice_semantics(): def reference_merge(previous_display, previous_context, result_messages, msg_text): # Faithful re-implementation of the PRE-optimization inner loop using the - # original `in context_keys[_cursor:]` list-slice membership test. + # original `in context_keys[_cursor:]` list-slice membership test. Apply + # the production replay reducer before and after that loop so this oracle + # isolates only the count-dict optimization under the current strict + # exact-payload contract. previous_display = list(previous_display or []) - _partial_seen = set() - _deduped_rev = [] - for m in reversed(previous_display): - if isinstance(m, dict) and m.get("_partial"): - key = streaming._message_identity(m) - if key is not None: - if key in _partial_seen: - continue - _partial_seen.add(key) - _deduped_rev.append(m) - previous_display = list(reversed(_deduped_rev)) previous_context = list(previous_context or []) result_messages = list(result_messages or []) + previous_display, _ = streaming._collapse_replayed_assistant_rows( + previous_display + ) + previous_context, _ = streaming._collapse_replayed_assistant_rows( + previous_context + ) if not result_messages: return previous_display if previous_display and previous_context: - _display_id_set = {streaming._message_identity(m) for m in previous_display} - _context_id_set = { - streaming._message_identity(m) - for m in previous_context - if not streaming._is_context_compression_marker(m) + display_keys, context_keys = streaming._display_backfill_projection_keys( + previous_display, + previous_context, + ) + display_counts = {} + context_counts = {} + for key in display_keys: + display_counts[key] = display_counts.get(key, 0) + 1 + for key in context_keys: + context_counts[key] = context_counts.get(key, 0) + 1 + insert_budget = { + key: count - display_counts.get(key, 0) + for key, count in context_counts.items() + if count > display_counts.get(key, 0) } - if bool(_context_id_set - _display_id_set): - context_keys = [streaming._message_identity(m) for m in previous_context] + if insert_budget: _backfilled = [] - _context_inserted = set() _cursor = 0 - for _di, _dmsg in enumerate(previous_display): - _dkey = streaming._message_identity(_dmsg) - if _dkey is not None: - _j = _cursor - while _j < len(context_keys) and context_keys[_j] != _dkey: - _j += 1 - if _j < len(context_keys): - for _k in range(_cursor, _j): - _ckey = context_keys[_k] - _cmsg = previous_context[_k] - if _ckey is not None and _ckey not in _context_inserted and _ckey not in _display_id_set and not streaming._is_context_compression_marker(_cmsg): - _backfilled.append(_copy.deepcopy(_cmsg)) - _context_inserted.add(_ckey) - _cursor = _j + 1 - elif not any( - streaming._message_identity(_f) in context_keys[_cursor:] - for _f in previous_display[_di + 1:] + + def backfill_range(start, stop): + for index in range(start, stop): + key = context_keys[index] + message = previous_context[index] + if ( + insert_budget.get(key, 0) > 0 + and not streaming._is_context_compression_marker(message) + and not streaming._is_compressed_context_tool_result_summary_message(message) ): - for _k in range(_cursor, len(context_keys)): - _ckey = context_keys[_k] - _cmsg = previous_context[_k] - if _ckey is not None and _ckey not in _context_inserted and _ckey not in _display_id_set and not streaming._is_context_compression_marker(_cmsg): - _backfilled.append(_copy.deepcopy(_cmsg)) - _context_inserted.add(_ckey) - _cursor = len(context_keys) + _backfilled.append(_copy.deepcopy(message)) + insert_budget[key] -= 1 + + for _di, _dmsg in enumerate(previous_display): + _dkey = display_keys[_di] + _j = _cursor + while _j < len(context_keys) and context_keys[_j] != _dkey: + _j += 1 + if _j < len(context_keys): + backfill_range(_cursor, _j) + _cursor = _j + 1 + elif not any( + future_key in context_keys[_cursor:] + for future_key in display_keys[_di + 1:] + ): + backfill_range(_cursor, len(context_keys)) + _cursor = len(context_keys) _backfilled.append(_dmsg) - while _cursor < len(context_keys): - _ckey = context_keys[_cursor] - _cmsg = previous_context[_cursor] - _cursor += 1 - if _ckey is not None and _ckey not in _context_inserted and _ckey not in _display_id_set and not streaming._is_context_compression_marker(_cmsg): - _backfilled.append(_copy.deepcopy(_cmsg)) - _context_inserted.add(_ckey) + backfill_range(_cursor, len(context_keys)) if len(_backfilled) > len(previous_display): previous_display = _backfilled # Both share the identical tail-merge logic after backfill; compare backfill output. + previous_display, _ = streaming._collapse_replayed_assistant_rows( + previous_display + ) return previous_display rng = random.Random(2026) diff --git a/tests/test_replay_payload_strictness.py b/tests/test_replay_payload_strictness.py new file mode 100644 index 00000000000..f17f1405847 --- /dev/null +++ b/tests/test_replay_payload_strictness.py @@ -0,0 +1,1324 @@ +import copy +import contextlib +import json +from types import SimpleNamespace + +import pytest + + +class _IntSubclass(int): + pass + + +class _StrSubclass(str): + pass + + +def _empty_assistant( + message_id: object = "assistant-a", *, finish_reason="stop", reasoning="same" +): + return { + "role": "assistant", + "content": "", + "id": message_id, + "finish_reason": finish_reason, + "reasoning": reasoning, + "timestamp": 123, + } + + +def _structured_assistant(image_url): + return { + "role": "assistant", + "content": [{"type": "image_url", "image_url": image_url}], + "finish_reason": "incomplete", + } + + +def _session_payload(tmp_path, sid, *, messages, context_messages=None, **extra): + payload = { + "session_id": sid, + "title": "strict replay repair", + "workspace": str(tmp_path), + "model": "test-model", + "created_at": 100.0, + "updated_at": 200.0, + "messages": messages, + "tool_calls": [], + } + if context_messages is not None: + payload["context_messages"] = context_messages + payload.update(extra) + return payload + + +def _patch_store(monkeypatch, models, session_dir): + session_dir.mkdir(exist_ok=True) + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", session_dir / "_index.json") + + +def test_canonical_digest_is_type_faithful(): + from api.models import _canonical_message_digest + + tuple_payload = {"role": "assistant", "content": "", "meta": (1, 2)} + list_payload = {"role": "assistant", "content": "", "meta": [1, 2]} + int_key_payload = {"role": "assistant", "content": "", "meta": {1: "v"}} + str_key_payload = {"role": "assistant", "content": "", "meta": {"1": "v"}} + + assert _canonical_message_digest(tuple_payload) is None + assert _canonical_message_digest(int_key_payload) is None + assert _canonical_message_digest(list_payload) is not None + assert _canonical_message_digest(str_key_payload) is not None + + +def test_incomplete_reducer_requires_identical_full_payload(): + from api.models import _collapse_duplicate_incomplete_message_ids + + first = _empty_assistant(1701, finish_reason="incomplete", reasoning="alpha") + distinct = _empty_assistant(1701, finish_reason="incomplete", reasoning="beta") + duplicate = copy.deepcopy(first) + + collapsed, changed = _collapse_duplicate_incomplete_message_ids( + [first, distinct, duplicate] + ) + + assert changed is True + assert collapsed == [first, distinct] + + +@pytest.mark.parametrize( + "content", + [ + [{"type": "text", "text": {}}], + [{"type": "text", "text": 0}], + [{"type": "text", "text": []}], + [{"type": "text"}], + [{"type": "text", "text": "", "content": "hidden alternative"}], + [{"type": "text", "text": "", "image_url": "file:///A.png"}], + ], +) +def test_empty_text_schema_rejects_ambiguous_blocks(content): + from api.models import _is_admissible_empty_text_content + + assert _is_admissible_empty_text_content(content) is False + + +def test_replay_fallback_preserves_distinct_nontext_assistants(): + from api.streaming import _deduplicate_context_messages, _message_replay_key + + first = { + "role": "assistant", + "content": [{"type": "image_url", "image_url": "file:///A.png"}], + "finish_reason": "incomplete", + "id": 1, + } + second = { + "role": "assistant", + "content": [{"type": "image_url", "image_url": "file:///B.png"}], + "finish_reason": "incomplete", + "id": 2, + } + + assert _message_replay_key(first) is None + assert _message_replay_key(second) is None + assert _deduplicate_context_messages([first, second]) == [first, second] + + +def test_replay_prefix_does_not_strip_noncomparable_structured_assistants(): + from api.streaming import _message_replay_key, _strip_replayed_prefix + + first = _structured_assistant("file:///A.png") + second = _structured_assistant("file:///B.png") + candidate_tail = {"role": "assistant", "content": "keep the tail"} + + assert _message_replay_key(first) is None + assert _message_replay_key(second) is None + assert _strip_replayed_prefix([first], [second, candidate_tail]) == [ + second, + candidate_tail, + ] + + +def test_replayed_context_block_requires_all_comparison_keys_to_be_conclusive(): + from api.streaming import _message_replay_key, _strip_replayed_context_items + + first = _structured_assistant("file:///A.png") + second = _structured_assistant("file:///B.png") + shared_tail = [ + {"role": "user", "content": "shared user row"}, + {"role": "assistant", "content": "shared assistant row"}, + ] + existing = [first, *shared_tail] + candidates = [second, *copy.deepcopy(shared_tail)] + + assert len(existing) == len(candidates) == 3 + assert _message_replay_key(first) is None + assert _message_replay_key(second) is None + assert _strip_replayed_context_items(existing, candidates) == candidates + + +def test_messages_prefix_rejects_noncomparable_structured_assistants(): + from api.streaming import _message_replay_key, _messages_have_prefix + + first = _structured_assistant("file:///A.png") + second = _structured_assistant("file:///B.png") + + assert _message_replay_key(first) is None + assert _message_replay_key(second) is None + assert not _messages_have_prefix([first], [second], key_fn=_message_replay_key) + + +def test_default_prefix_accepts_only_exact_structured_payload(): + from api.streaming import _messages_have_prefix + + first = _structured_assistant("file:///A.png") + identical = copy.deepcopy(first) + different = _structured_assistant("file:///B.png") + + assert _messages_have_prefix([identical], [first]) is False + assert _messages_have_prefix([identical], [first], allow_exact_payload=True) is True + assert ( + _messages_have_prefix([different], [first], allow_exact_payload=True) is False + ) + + +def test_exact_prefix_mode_rejects_lossy_structured_identity_matches(): + from api.streaming import ( + _message_replay_key, + _messages_have_prefix, + ) + + first = { + "role": "assistant", + "content": [ + {"type": "text", "text": "same visible text"}, + {"type": "image_url", "image_url": "file:///A.png", "detail": True}, + ], + } + different_image = copy.deepcopy(first) + different_image["content"][1]["image_url"] = "file:///B.png" + different_scalar_type = copy.deepcopy(first) + different_scalar_type["content"][1]["detail"] = 1 + + assert _message_replay_key(first) is None + assert _message_replay_key(different_image) is None + assert not _messages_have_prefix( + [copy.deepcopy(first)], [first], key_fn=_message_replay_key + ) + assert not _messages_have_prefix( + [different_image], [first], allow_exact_payload=True + ) + assert not _messages_have_prefix( + [different_scalar_type], [first], allow_exact_payload=True + ) + + +def test_exact_prefix_mode_rejects_container_subclasses(): + from api.streaming import _message_replay_key, _messages_have_prefix + + class MessageDict(dict): + pass + + first = {"role": "assistant", "content": "same visible text"} + subclassed = MessageDict(copy.deepcopy(first)) + + assert _message_replay_key(subclassed) is None + assert not _messages_have_prefix([subclassed], [first], key_fn=_message_replay_key) + assert not _messages_have_prefix([subclassed], [first], allow_exact_payload=True) + + +def test_exact_prefix_mode_requires_identical_nonempty_attachments(): + from api.streaming import _message_replay_key, _messages_have_prefix + + first = { + "role": "user", + "content": "same visible text", + "attachments": [{"name": "A.pdf"}], + } + identical = copy.deepcopy(first) + different = copy.deepcopy(first) + different["attachments"][0]["name"] = "B.pdf" + + assert _message_replay_key(first) is None + assert not _messages_have_prefix([identical], [first], key_fn=_message_replay_key) + assert not _messages_have_prefix([identical], [first]) + assert _messages_have_prefix([identical], [first], allow_exact_payload=True) + assert not _messages_have_prefix([different], [first], allow_exact_payload=True) + + +def test_authoritative_prefix_rejects_different_durable_ids(): + from api.streaming import _result_has_authoritative_full_history_prefix + + previous = [{"role": "user", "content": "old prompt", "id": "old-user"}] + result = [ + {"role": "user", "content": "old prompt", "id": "different-user"}, + {"role": "assistant", "content": "new answer"}, + ] + + assert not _result_has_authoritative_full_history_prefix( + result, + previous, + { + "text": "current prompt", + "current_turn_user_idx": len(previous), + "turn_id": "turn:durable-id-control", + }, + "current prompt", + ) + + +def _settle_structured_result(previous, result): + from api.streaming import _settle_result_messages + + prompt = "continue the active turn" + session = SimpleNamespace( + session_id="strict-structured-prefix", + messages=copy.deepcopy(previous), + context_messages=copy.deepcopy(previous), + truncation_watermark=None, + ) + _settle_result_messages( + session, + copy.deepcopy(previous), + copy.deepcopy(previous), + copy.deepcopy(result), + prompt, + "webui", + { + "token": "stream:strict-structured-prefix", + "text": prompt, + "timestamp": 200.0, + "source": "webui", + "attachments": [], + "current_turn_user_idx": len(previous), + "turn_id": "turn:strict-structured-prefix", + "agent_turn_boundary_resolved": True, + "agent_turn_boundary_source": "result", + }, + ) + return session + + +def test_settle_legacy_double_exact_history_replay_is_idempotent(): + from api.streaming import _settle_result_messages + + previous = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + prompt = "continue the active turn" + result = [ + *copy.deepcopy(previous), + *copy.deepcopy(previous), + {"role": "user", "content": prompt}, + {"role": "assistant", "content": "new answer"}, + ] + session = SimpleNamespace( + session_id="strict-legacy-double-replay", + messages=copy.deepcopy(previous), + context_messages=copy.deepcopy(previous), + truncation_watermark=None, + ) + + _settle_result_messages( + session, + copy.deepcopy(previous), + copy.deepcopy(previous), + result, + prompt, + "webui", + { + "token": None, + "text": prompt, + "timestamp": None, + "source": "webui", + "attachments": [], + "current_turn_user_idx": None, + "turn_id": "", + }, + ) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + ] + assert [row.get("content") for row in projection] == [ + "old prompt", + "old answer", + prompt, + "new answer", + ] + + +def test_settle_preserves_exact_structured_assistant_delta(): + previous = [_structured_assistant("file:///A.png")] + previous[0]["id"] = 114 + repeated_delta = copy.deepcopy(previous[0]) + answer = {"role": "assistant", "content": "new answer"} + + session = _settle_structured_result(previous, [repeated_delta, answer]) + + assert len(session.messages) == 4 + assert len(session.context_messages) == 3 + assert sum(isinstance(row.get("content"), list) for row in session.messages) == 2 + assert ( + sum(isinstance(row.get("content"), list) for row in session.context_messages) + == 1 + ) + for projection in (session.messages, session.context_messages): + assert sum(row.get("role") == "user" for row in projection) == 1 + assert projection[-1]["content"] == "new answer" + + +def test_settle_strips_exact_structured_prefix_from_full_history(): + previous = [_structured_assistant("file:///A.png")] + previous[0]["id"] = "structured-assistant-a" + current_user = {"role": "user", "content": "continue the active turn"} + answer = {"role": "assistant", "content": "new answer"} + + session = _settle_structured_result(previous, [*previous, current_user, answer]) + + assert len(session.messages) == len(session.context_messages) == 3 + for projection in (session.messages, session.context_messages): + assert sum(isinstance(row.get("content"), list) for row in projection) == 1 + assert [row.get("role") for row in projection] == [ + "assistant", + "user", + "assistant", + ] + assert projection[-1]["content"] == "new answer" + + +def test_settle_strips_exact_structured_history_with_out_of_band_current_user(): + previous = [ + {"role": "user", "content": "old prompt", "id": 101}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "old answer", "annotations": ["durable"]} + ], + "id": 102, + "reasoning": "durable reasoning", + "api_content": "durable provider payload", + }, + ] + answer = {"role": "assistant", "content": "new answer", "id": 103} + + session = _settle_structured_result(previous, [*copy.deepcopy(previous), answer]) + + for projection in (session.messages, session.context_messages): + assert len(projection) == 4 + assert projection[:2] == previous + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + ] + assert projection[2]["content"] == "continue the active turn" + assert projection[3] == answer + + +def test_settle_preserves_idless_exact_prefix_authority_after_id_assignment(): + previous = [ + {"role": "user", "content": "old prompt"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "old answer", "annotations": ["durable"]} + ], + "reasoning": "durable reasoning", + }, + ] + answer = {"role": "assistant", "content": "new answer"} + + session = _settle_structured_result(previous, [*copy.deepcopy(previous), answer]) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + ] + assert [row.get("content") for row in projection].count("old prompt") == 1 + assert ( + sum( + isinstance(row.get("content"), list) + and row["content"][0].get("text") == "old answer" + for row in projection + ) + == 1 + ) + assert projection[-1]["content"] == "new answer" + assert type(projection[-1].get("id")) is int + + +def test_settle_preserves_payload_distinct_rejected_history_prefix(): + previous = [ + {"role": "user", "content": "old prompt", "id": 101}, + { + "role": "assistant", + "content": "same answer", + "id": 102, + "reasoning": "old reasoning", + "model": "old-model", + "request_id": "old-request", + }, + ] + distinct_history = { + "role": "assistant", + "content": "same answer", + "id": 104, + "reasoning": "distinct reasoning", + "model": "distinct-model", + "request_id": "distinct-request", + } + final = { + "role": "assistant", + "content": "final answer", + "id": 105, + } + + session = _settle_structured_result( + previous, + [copy.deepcopy(previous[0]), distinct_history, final], + ) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + "assistant", + ] + assert [ + row.get("id") for row in projection if row.get("role") == "assistant" + ] == [102, 104, 105] + assert projection[1] == previous[1] + assert projection[2]["content"] == "continue the active turn" + assert projection[2]["_active_turn_token"] == ( + "stream:strict-structured-prefix" + ) + assert type(projection[2].get("id")) is int + assert projection[3] == distinct_history + assert session.messages[2]["id"] == session.context_messages[2]["id"] + + +def test_sync_chat_uses_strict_turn_provenance_for_rejected_history_prefix( + monkeypatch, + tmp_path, +): + """The synchronous route must not re-enable visible-only prefix deletion.""" + from api import config, routes + + previous = [ + {"role": "user", "content": "old prompt", "id": 101}, + { + "role": "assistant", + "content": "same answer", + "id": 102, + "reasoning": "old reasoning", + "model": "old-model", + "request_id": "old-request", + }, + ] + distinct_history = { + "role": "assistant", + "content": "same answer", + "id": 104, + "reasoning": "distinct reasoning", + "model": "distinct-model", + "request_id": "distinct-request", + } + prompt = "continue the active turn" + final = { + "role": "assistant", + "content": "final answer", + "id": 105, + } + result = { + "messages": [ + copy.deepcopy(previous[0]), + copy.deepcopy(distinct_history), + {"role": "user", "content": prompt}, + copy.deepcopy(final), + ], + "current_turn_user_idx": len(previous), + "turn_id": "turn:sync-strict-prefix", + "final_response": "final answer", + "completed": True, + } + + class _Session: + session_id = "sync-strict-prefix" + workspace = str(tmp_path) + model = "test-model" + model_provider = "test-provider" + profile = "default" + pending_user_source = "webui" + title = "Already titled" + input_tokens = 0 + output_tokens = 0 + estimated_cost = 0.0 + cache_read_tokens = 0 + cache_write_tokens = 0 + truncation_watermark = None + messages = copy.deepcopy(previous) + context_messages = copy.deepcopy(previous) + + def save(self): + return None + + def compact(self): + return { + "session_id": self.session_id, + "title": self.title, + "message_count": len(self.messages), + } + + class _Agent: + def __init__(self, **_kwargs): + self._persist_user_message_idx = len(previous) + self._current_turn_id = "turn:sync-strict-prefix" + + def run_conversation(self, **_kwargs): + return copy.deepcopy(result) + + session = _Session() + monkeypatch.setattr( + routes, "_agent_runtime_barrier_response", lambda **_kwargs: None + ) + monkeypatch.setattr(routes, "_session_is_subagent_view_only", lambda _sid: False) + monkeypatch.setattr(routes, "get_session", lambda _sid: session) + monkeypatch.setattr(routes, "resolve_trusted_workspace", lambda value: value) + monkeypatch.setattr( + routes, "_get_session_agent_lock", lambda _sid: contextlib.nullcontext() + ) + monkeypatch.setattr( + routes, "_read_profile_model_config", lambda *_args: (None, None, {}) + ) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda *_args, **_kwargs: ("test-model", "test-provider"), + ) + monkeypatch.setattr(routes, "require_ai_agent_class", lambda: _Agent) + monkeypatch.setattr(routes, "_resolve_cli_toolsets", lambda: []) + monkeypatch.setattr(routes, "get_config", lambda: {}) + monkeypatch.setattr(routes, "load_settings", lambda: {}) + monkeypatch.setattr(routes, "title_from", lambda _messages, fallback: fallback) + monkeypatch.setattr(routes, "public_session_projection", lambda payload: payload) + monkeypatch.setattr(routes, "j", lambda _handler, payload, status=200: payload) + monkeypatch.setattr( + config, + "resolve_model_provider", + lambda _model: ("test-model", "test-provider", None), + ) + + routes._handle_chat_sync( + object(), + { + "session_id": session.session_id, + "message": prompt, + "workspace": str(tmp_path), + }, + ) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + "assistant", + ] + assert [ + row.get("id") for row in projection if row.get("role") == "assistant" + ] == [102, 104, 105] + assert projection[2]["content"] == prompt + assert type(projection[2].get("id")) is int + assert projection[3] == distinct_history + assert session.messages[2]["id"] == session.context_messages[2]["id"] + + +def test_display_backfill_preserves_payload_distinct_same_visible_assistant(): + from api.streaming import _merge_display_messages_after_agent_result + + user = {"role": "user", "content": "old prompt", "id": "old-user"} + visible = { + "role": "assistant", + "content": "same answer", + "id": "assistant-a", + "reasoning": "first reasoning", + } + context_only = { + "role": "assistant", + "content": "same answer", + "id": "assistant-b", + "reasoning": "second reasoning", + } + current = {"role": "user", "content": "next prompt", "id": "current-user"} + final = {"role": "assistant", "content": "done", "id": "assistant-final"} + previous_display = [user, visible] + previous_context = [user, visible, context_only] + + merged = _merge_display_messages_after_agent_result( + copy.deepcopy(previous_display), + copy.deepcopy(previous_context), + copy.deepcopy(previous_context + [current, final]), + "next prompt", + verification_nudge_provenance={ + "active_turn_identity": { + "token": "stream:backfill", + "text": "next prompt", + "current_turn_user_idx": len(previous_context), + "turn_id": "turn:backfill", + "agent_turn_boundary_resolved": True, + "agent_turn_boundary_source": "result", + } + }, + ) + + assert [row.get("id") for row in merged] == [ + "old-user", + "assistant-a", + "assistant-b", + "current-user", + "assistant-final", + ] + assert merged[1] == visible + assert merged[2] == context_only + + +def test_display_merge_collapses_only_exact_durable_empty_replay(): + from api.streaming import _merge_display_messages_after_agent_result + + prompt = "Continue the active turn." + token = "stream:active-turn" + active_user = { + "role": "user", + "content": prompt, + "_active_turn_token": token, + } + replay = _empty_assistant("durable-empty") + previous = [active_user, replay] + provenance = { + "active_turn_identity": { + "token": token, + "text": prompt, + "current_turn_user_idx": 0, + "turn_id": "turn-active", + "agent_turn_boundary_resolved": True, + "agent_turn_boundary_source": "result", + } + } + + merged = _merge_display_messages_after_agent_result( + previous, + previous, + previous + [copy.deepcopy(replay)], + prompt, + verification_nudge_provenance=provenance, + ) + assert merged == previous + + distinct = copy.deepcopy(replay) + distinct["reasoning"] = "different" + merged_distinct = _merge_display_messages_after_agent_result( + previous, + previous, + previous + [distinct], + prompt, + verification_nudge_provenance=provenance, + ) + assert merged_distinct == previous + [distinct] + + synthetic_only = _merge_display_messages_after_agent_result( + previous + [copy.deepcopy(replay)], + previous + [copy.deepcopy(replay)], + [ + { + "role": "user", + "content": "[System: verify the workspace]", + "_verification_stop_synthetic": True, + } + ], + prompt, + verification_nudge_provenance=provenance, + ) + assert synthetic_only == previous + + +@pytest.mark.parametrize( + ("first", "distinct"), + [ + ( + {"role": "assistant", "content": "same", "id": "assistant-a"}, + {"role": "assistant", "content": "same", "id": "assistant-b"}, + ), + ( + { + "role": "assistant", + "content": "same", + "attachments": [{"name": "A.pdf"}], + }, + { + "role": "assistant", + "content": "same", + "attachments": [{"name": "B.pdf"}], + }, + ), + ( + {"role": "assistant", "content": "same", "reasoning": "alpha"}, + {"role": "assistant", "content": "same", "reasoning": "beta"}, + ), + ( + {"role": "assistant", "content": "same", "model": "model-a"}, + {"role": "assistant", "content": "same", "model": "model-b"}, + ), + ( + {"role": "assistant", "content": "same", "request_id": "request-a"}, + {"role": "assistant", "content": "same", "request_id": "request-b"}, + ), + ( + { + "role": "assistant", + "content": [{"type": "text", "text": "same", "annotations": ["A"]}], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "same", "annotations": ["B"]}], + }, + ), + ], +) +def test_display_merge_preserves_distinct_structured_nonempty_assistants( + first, + distinct, +): + from api.streaming import _merge_display_messages_after_agent_result + + prompt = "Continue the active turn." + + merged = _merge_display_messages_after_agent_result( + [], + [], + [first, distinct], + prompt, + ) + + assistants = [row for row in merged if row.get("role") == "assistant"] + assert assistants == [first, distinct] + + +def test_display_merge_collapses_exact_nonempty_assistant_payload(): + from api.streaming import _merge_display_messages_after_agent_result + + first = { + "role": "assistant", + "content": "same", + "id": "assistant-a", + "reasoning": "same reasoning", + "attachments": [{"name": "A.pdf"}], + } + + merged = _merge_display_messages_after_agent_result( + [], + [], + [first, copy.deepcopy(first)], + "Continue the active turn.", + ) + + assistants = [row for row in merged if row.get("role") == "assistant"] + assert assistants == [first] + + +def test_settle_keeps_payload_distinct_assistants_in_both_projections(): + first = { + "role": "assistant", + "content": "same", + "id": 106, + "reasoning": "first reasoning", + "annotations": [{"source": "A"}], + } + distinct = { + **copy.deepcopy(first), + "id": 107, + "reasoning": "second reasoning", + "annotations": [{"source": "B"}], + } + + session = _settle_structured_result([], [first, distinct]) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "assistant", + ] + assert [ + row.get("id") for row in projection if row.get("role") == "assistant" + ] == [106, 107] + + +def test_settle_preserves_payload_distinct_assistant_only_suffix(): + previous = [ + {"role": "user", "content": "old prompt", "id": 101}, + { + "role": "assistant", + "content": "same visible answer", + "id": 109, + "reasoning": "old reasoning", + }, + ] + distinct = { + "role": "assistant", + "content": "same visible answer", + "id": 108, + "reasoning": "current reasoning", + } + + session = _settle_structured_result(previous, [distinct]) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + ] + assert [ + row.get("id") for row in projection if row.get("role") == "assistant" + ] == [109, 108] + assert projection[-1] == distinct + + +def test_settle_preserves_payload_distinct_repeated_context_block(): + previous = [ + {"role": "user", "content": "old prompt", "id": 101}, + { + "role": "assistant", + "content": "same block answer", + "id": 109, + "reasoning": "old reasoning", + }, + { + "role": "tool", + "content": "shared tool result", + "tool_call_id": "call-shared", + "id": 113, + }, + { + "role": "assistant", + "content": "shared block tail", + "id": 112, + }, + { + "role": "assistant", + "content": "history barrier", + "id": 110, + }, + ] + repeated_block = [ + { + "role": "assistant", + "content": "same block answer", + "id": 108, + "reasoning": "current reasoning", + }, + copy.deepcopy(previous[2]), + copy.deepcopy(previous[3]), + ] + final = { + "role": "assistant", + "content": "current final", + "id": 111, + } + + session = _settle_structured_result(previous, [*repeated_block, final]) + + for projection in (session.messages, session.context_messages): + current_user_idx = next( + idx + for idx, row in enumerate(projection) + if row.get("_active_turn_token") == "stream:strict-structured-prefix" + ) + assert projection[current_user_idx + 1 :] == [*repeated_block, final] + + +def test_synthetic_only_settle_repairs_live_and_persisted_projections( + tmp_path, + monkeypatch, +): + from api import models + from api.streaming import _settle_result_messages + + session_dir = tmp_path / "sessions" + _patch_store(monkeypatch, models, session_dir) + prompt = "Continue the active turn." + token = "stream:synthetic-only" + active_user = { + "role": "user", + "content": prompt, + "timestamp": 200.0, + "_active_turn_token": token, + } + replay = _empty_assistant("durable-empty") + duplicate_history = [active_user, replay, copy.deepcopy(replay)] + session = models.Session( + session_id="synthetic-only-live-persisted-parity", + title="synthetic-only live/persisted parity", + workspace=str(tmp_path), + model="test-model", + messages=copy.deepcopy(duplicate_history), + context_messages=copy.deepcopy(duplicate_history), + created_at=100.0, + updated_at=200.0, + ) + + _settle_result_messages( + session, + copy.deepcopy(duplicate_history), + copy.deepcopy(duplicate_history), + [ + { + "role": "user", + "content": "[System: verify the workspace]", + "_verification_stop_synthetic": True, + } + ], + prompt, + "webui", + { + "token": token, + "text": prompt, + "timestamp": 200.0, + "source": "webui", + "attachments": [], + "current_turn_user_idx": 0, + "turn_id": "turn:synthetic-only", + "agent_turn_boundary_resolved": True, + "agent_turn_boundary_source": "result", + }, + ) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == ["user", "assistant"] + assert [row for row in projection if row.get("role") == "assistant"] == [replay] + + session.save(skip_index=True, touch_updated_at=False) + persisted = json.loads( + (session_dir / f"{session.session_id}.json").read_text(encoding="utf-8") + ) + loaded = models.Session.load(session.session_id) + + assert loaded is not None + for projection in ( + persisted["messages"], + persisted["context_messages"], + loaded.messages, + loaded.context_messages, + ): + assert [row.get("role") for row in projection] == ["user", "assistant"] + assert len([row for row in projection if row.get("role") == "assistant"]) == 1 + + +def test_settle_collapses_exact_assistant_payload_in_both_projections(): + first = { + "role": "assistant", + "content": "same", + "id": 106, + "reasoning": "same reasoning", + "attachments": [{"name": "A.pdf"}], + } + + session = _settle_structured_result([], [first, copy.deepcopy(first)]) + + for projection in (session.messages, session.context_messages): + assistants = [row for row in projection if row.get("role") == "assistant"] + assert assistants == [first] + + +def test_settle_collapses_idless_exact_assistant_before_stable_id_assignment(): + first = { + "role": "assistant", + "content": "same", + "reasoning": "same reasoning", + "attachments": [{"name": "A.pdf"}], + } + + session = _settle_structured_result([], [first, copy.deepcopy(first)]) + + for projection in (session.messages, session.context_messages): + assistants = [row for row in projection if row.get("role") == "assistant"] + assert len(assistants) == 1 + assert assistants[0]["content"] == "same" + assert type(assistants[0].get("id")) is int + + +def test_settle_preserves_idless_repeated_answer_across_current_turn_boundary(): + previous = [ + {"role": "user", "content": "Say it once."}, + {"role": "assistant", "content": "same"}, + ] + result = [*copy.deepcopy(previous), {"role": "assistant", "content": "same"}] + + session = _settle_structured_result(previous, result) + + for projection in (session.messages, session.context_messages): + assert [row.get("role") for row in projection] == [ + "user", + "assistant", + "user", + "assistant", + ] + assert [ + row.get("content") for row in projection if row.get("role") == "assistant" + ] == ["same", "same"] + assistant_ids = [ + row.get("id") for row in projection if row.get("role") == "assistant" + ] + assert type(assistant_ids[-1]) is int + if assistant_ids[0] is not None: + assert assistant_ids[0] != assistant_ids[-1] + + +@pytest.mark.parametrize( + "bad_idx", + [True, 1.0, 1.5, "1", _IntSubclass(1), None, -1], +) +def test_active_turn_authority_rejects_lossy_index_types(bad_idx): + from api.streaming import ( + _active_turn_boundary_is_valid, + _resolve_active_turn_authority, + ) + + resolved = _resolve_active_turn_authority( + {"current_turn_user_idx": None, "turn_id": ""}, + result={"current_turn_user_idx": bad_idx, "turn_id": "turn-1"}, + ) + + assert resolved["current_turn_user_idx"] is None + assert _active_turn_boundary_is_valid(resolved) is False + + +@pytest.mark.parametrize( + "bad_turn_id", + [None, "", " ", 1, ["turn-1"], {"turn": "1"}, _StrSubclass("turn-1")], +) +def test_active_turn_authority_rejects_lossy_turn_id_types(bad_turn_id): + from api.streaming import ( + _active_turn_boundary_is_valid, + _resolve_active_turn_authority, + ) + + resolved = _resolve_active_turn_authority( + {"current_turn_user_idx": None, "turn_id": ""}, + result={"current_turn_user_idx": 1, "turn_id": bad_turn_id}, + ) + + assert resolved["current_turn_user_idx"] is None + assert resolved["turn_id"] == "" + assert _active_turn_boundary_is_valid(resolved) is False + + +def test_active_turn_authority_does_not_mix_fields_across_attempts(): + from api.streaming import ( + _active_turn_boundary_is_valid, + _resolve_active_turn_authority, + ) + + resolved = _resolve_active_turn_authority( + { + "token": "stream:attempt", + "text": "prompt", + "current_turn_user_idx": 1, + "turn_id": "turn:first-attempt", + }, + result={"current_turn_user_idx": 2, "turn_id": []}, + agent=SimpleNamespace( + _persist_user_message_idx=None, + _current_turn_id="turn:second-attempt", + ), + ) + + assert resolved["current_turn_user_idx"] is None + assert resolved["turn_id"] == "" + assert _active_turn_boundary_is_valid(resolved) is False + + +def test_active_turn_authority_accepts_complete_result_pair_atomically(): + from api.streaming import _resolve_active_turn_authority + + resolved = _resolve_active_turn_authority( + {"current_turn_user_idx": 1, "turn_id": "turn:first-attempt"}, + result={"current_turn_user_idx": 2, "turn_id": "turn:result"}, + agent=SimpleNamespace( + _persist_user_message_idx=3, + _current_turn_id="turn:agent", + ), + ) + + assert resolved["current_turn_user_idx"] == 2 + assert resolved["turn_id"] == "turn:result" + + +def test_active_turn_authority_falls_back_to_complete_agent_pair(): + from api.streaming import _resolve_active_turn_authority + + resolved = _resolve_active_turn_authority( + {"current_turn_user_idx": 1, "turn_id": "turn:first-attempt"}, + result={"current_turn_user_idx": 2, "turn_id": []}, + agent=SimpleNamespace( + _persist_user_message_idx=3, + _current_turn_id="turn:agent", + ), + ) + + assert resolved["current_turn_user_idx"] == 3 + assert resolved["turn_id"] == "turn:agent" + + +def test_partial_reducer_only_removes_identical_rows(): + from api.models import _collapse_adjacent_duplicate_partials + + first = { + **_empty_assistant(1701, finish_reason="incomplete"), + "_partial": True, + "attachments": [{"name": "A.pdf"}], + } + different_id = {**copy.deepcopy(first), "id": 1702} + different_attachment = copy.deepcopy(first) + different_attachment["attachments"] = [{"name": "B.pdf"}] + + collapsed, changed = _collapse_adjacent_duplicate_partials( + [first, different_id, different_attachment, copy.deepcopy(different_attachment)] + ) + + assert changed is True + assert collapsed == [first, different_id, different_attachment] + + +def test_non_incomplete_partial_replay_key_uses_exact_payload_digest(): + from api.streaming import _message_replay_key + + first = { + "role": "assistant", + "content": "", + "id": "partial-a", + "_partial": True, + "reasoning": "working", + "timestamp": 123, + } + exact = copy.deepcopy(first) + distinct = copy.deepcopy(first) + distinct["reasoning"] = "different" + + assert _message_replay_key(first) == _message_replay_key(exact) + assert _message_replay_key(first) != _message_replay_key(distinct) + + +def test_load_repairs_messages_and_context_with_one_pipeline(tmp_path, monkeypatch): + from api import models + + session_dir = tmp_path / "sessions" + _patch_store(monkeypatch, models, session_dir) + sid = "strict-context-parity" + replay = _empty_assistant("assistant-a") + payload = _session_payload( + tmp_path, + sid, + messages=[replay, copy.deepcopy(replay)], + context_messages=[copy.deepcopy(replay), copy.deepcopy(replay)], + ) + (session_dir / f"{sid}.json").write_text(json.dumps(payload), encoding="utf-8") + + loaded = models.Session.load(sid) + persisted = json.loads((session_dir / f"{sid}.json").read_text(encoding="utf-8")) + + assert loaded is not None + assert len(loaded.messages) == len(loaded.context_messages) == 1 + assert len(persisted["messages"]) == len(persisted["context_messages"]) == 1 + + +def test_load_preserves_distinct_nonempty_payloads_and_repairs_exact_replay( + tmp_path, + monkeypatch, +): + from api import models + + session_dir = tmp_path / "sessions" + _patch_store(monkeypatch, models, session_dir) + sid = "strict-nonempty-context-parity" + first = { + "role": "assistant", + "content": "same", + "id": "assistant-a", + "reasoning": "first", + } + distinct = { + **copy.deepcopy(first), + "id": "assistant-b", + "reasoning": "second", + } + payload = _session_payload( + tmp_path, + sid, + messages=[first, distinct, copy.deepcopy(distinct)], + context_messages=[ + copy.deepcopy(first), + copy.deepcopy(distinct), + copy.deepcopy(distinct), + ], + ) + (session_dir / f"{sid}.json").write_text(json.dumps(payload), encoding="utf-8") + + loaded = models.Session.load(sid) + persisted = json.loads((session_dir / f"{sid}.json").read_text(encoding="utf-8")) + + expected_ids = ["assistant-a", "assistant-b"] + assert loaded is not None + assert [row["id"] for row in loaded.messages] == expected_ids + assert [row["id"] for row in loaded.context_messages] == expected_ids + assert [row["id"] for row in persisted["messages"]] == expected_ids + assert [row["id"] for row in persisted["context_messages"]] == expected_ids + + +@pytest.mark.parametrize("kind", ["partial", "incomplete"]) +def test_any_visible_reduction_invalidates_positional_anchor( + tmp_path, monkeypatch, kind +): + from api import models + + session_dir = tmp_path / "sessions" + _patch_store(monkeypatch, models, session_dir) + sid = f"strict-anchor-{kind}" + if kind == "partial": + replay = {**_empty_assistant(1701), "_partial": True} + else: + replay = _empty_assistant(1701, finish_reason="incomplete") + payload = _session_payload( + tmp_path, + sid, + messages=[ + {"role": "user", "content": "before"}, + replay, + copy.deepcopy(replay), + {"role": "assistant", "content": "anchor"}, + ], + compression_anchor_visible_idx=3, + compression_anchor_message_key="stable-key", + ) + (session_dir / f"{sid}.json").write_text(json.dumps(payload), encoding="utf-8") + + loaded = models.Session.load(sid) + + assert loaded is not None + assert loaded.compression_anchor_visible_idx is None + assert loaded.compression_anchor_message_key == "stable-key" diff --git a/tests/test_sprint42.py b/tests/test_sprint42.py index a8b6d5cefeb..3ff019a8f14 100644 --- a/tests/test_sprint42.py +++ b/tests/test_sprint42.py @@ -933,15 +933,15 @@ def test_streaming_restores_prior_reasoning_metadata_after_followup(): "streaming.py must reinsert dropped reasoning-only assistant messages" -def test_routes_restores_prior_reasoning_metadata_after_followup(): - """The non-streaming route path must preserve prior reasoning metadata too.""" +def test_routes_delegates_prior_reasoning_metadata_restoration_to_shared_settlement(): + """The non-streaming route must use the same metadata-restoring settlement.""" src = (REPO / 'api' / 'routes.py').read_text(encoding="utf-8") - assert "_restore_reasoning_metadata" in src, \ - "routes.py must import reasoning metadata restoration helper" - assert "_next_context_messages" in src and "s.context_messages" in src, \ - "routes.py must restore prior reasoning metadata into model context" - assert 's.messages = _merge_display_messages_after_agent_result(' in src, \ - "routes.py must merge restored result messages into the visible transcript" + assert "_settle_result_messages" in src, \ + "routes.py must import the shared reasoning-restoring settlement helper" + assert "_previous_messages" in src and "_previous_context_messages" in src, \ + "routes.py must pass both prior projections to shared settlement" + assert '_settle_result_messages(' in src and '_result_messages' in src, \ + "routes.py must settle returned messages through the shared contract" class TestCredentialPoolBackwardCompat(unittest.TestCase): diff --git a/tests/test_webui_state_db_reconciliation.py b/tests/test_webui_state_db_reconciliation.py index 8077bf157fa..bf809e601f6 100644 --- a/tests/test_webui_state_db_reconciliation.py +++ b/tests/test_webui_state_db_reconciliation.py @@ -46,7 +46,7 @@ def _make_state_db(path: Path, sid: str, rows): "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, title TEXT, model TEXT, started_at REAL, message_count INTEGER)" ) conn.execute( - "CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT, role TEXT, content TEXT, timestamp REAL, tool_call_id TEXT, tool_calls TEXT, tool_name TEXT)" + "CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT, role TEXT, content TEXT, timestamp REAL, tool_call_id TEXT, tool_calls TEXT, tool_name TEXT, api_content TEXT)" ) conn.execute( "INSERT INTO sessions (id, source, title, model, started_at, message_count) VALUES (?, ?, ?, ?, ?, ?)", @@ -54,7 +54,7 @@ def _make_state_db(path: Path, sid: str, rows): ) for row in rows: conn.execute( - "INSERT INTO messages (session_id, role, content, timestamp, tool_call_id, tool_calls, tool_name) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO messages (session_id, role, content, timestamp, tool_call_id, tool_calls, tool_name, api_content) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( sid, row["role"], @@ -63,6 +63,7 @@ def _make_state_db(path: Path, sid: str, rows): row.get("tool_call_id"), row.get("tool_calls"), row.get("tool_name"), + row.get("api_content"), ), ) conn.commit() @@ -74,7 +75,7 @@ def _append_state_db_rows(path: Path, sid: str, rows): try: for row in rows: conn.execute( - "INSERT INTO messages (session_id, role, content, timestamp, tool_call_id, tool_calls, tool_name) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO messages (session_id, role, content, timestamp, tool_call_id, tool_calls, tool_name, api_content) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( sid, row["role"], @@ -83,6 +84,7 @@ def _append_state_db_rows(path: Path, sid: str, rows): row.get("tool_call_id"), row.get("tool_calls"), row.get("tool_name"), + row.get("api_content"), ), ) conn.execute( @@ -145,6 +147,193 @@ def _state_db_source_metadata(source): } +def _state_db_reconciliation_session(messages): + from types import SimpleNamespace + + return SimpleNamespace( + session_id="state-db-row-identity", + messages=messages, + truncation_watermark=None, + truncation_boundary=None, + ) + + +def _state_db_identity_message(row_id=...): + message = { + "role": "assistant", + "content": "same", + "timestamp": 1000.0, + "api_content": "same-wire", + } + if row_id is not ...: + message["_state_db_row_id"] = row_id + return message + + +def test_normal_state_db_reconciliation_preserves_distinct_sqlite_rows( + monkeypatch, + tmp_path, +): + """The public normal reader keeps source multiplicity but hides provenance.""" + import api.models as models + from api.helpers import public_session_projection + + sid = "normal_identical_state_rows" + _make_state_db( + tmp_path / "state.db", + sid, + [ + { + "role": "assistant", + "content": "same", + "timestamp": 1000.0, + "api_content": "same-wire", + }, + { + "role": "assistant", + "content": "same", + "timestamp": 1000.0, + "api_content": "same-wire", + }, + ], + ) + session = _install_test_session(monkeypatch, tmp_path, sid, []) + + source_rows = models.get_state_db_session_messages(sid) + reconciled = models.reconciled_state_db_messages_for_session(session) + + assert [message["_state_db_row_id"] for message in source_rows] == [1, 2] + assert [message["_state_db_row_id"] for message in reconciled] == [1, 2] + public = public_session_projection({"messages": reconciled}) + assert len(public["messages"]) == 2 + assert all( + not {"api_content", "_state_db_row_id", "_db_row_id", "state_db_row_id"} + & set(message) + for message in public["messages"] + ) + + +def test_normal_state_db_reconciliation_deduplicates_same_row_replay(): + import api.models as models + + replay = _state_db_identity_message(7) + reconciled = models.reconciled_state_db_messages_for_session( + _state_db_reconciliation_session([]), + state_messages=[replay, dict(replay)], + ) + + assert reconciled == [replay] + + +def test_normal_state_db_reconciliation_keeps_new_row_with_existing_sidecar(): + import api.models as models + + sidecar_row = _state_db_identity_message(1) + state_rows = [dict(sidecar_row), _state_db_identity_message(2)] + + reconciled = models.reconciled_state_db_messages_for_session( + _state_db_reconciliation_session([sidecar_row]), + state_messages=state_rows, + ) + + assert [message["_state_db_row_id"] for message in reconciled] == [1, 2] + + +@pytest.mark.parametrize( + ("first_row_id", "second_row_id"), + [ + (1, "1"), + (True, 1), + (True, "True"), + ], +) +def test_normal_state_db_reconciliation_keeps_typed_row_ids_distinct( + first_row_id, + second_row_id, +): + import api.models as models + + reconciled = models.reconciled_state_db_messages_for_session( + _state_db_reconciliation_session([]), + state_messages=[ + _state_db_identity_message(first_row_id), + _state_db_identity_message(second_row_id), + ], + ) + + assert [ + (type(message["_state_db_row_id"]), message["_state_db_row_id"]) + for message in reconciled + ] == [ + (type(first_row_id), first_row_id), + (type(second_row_id), second_row_id), + ] + + +def test_normal_state_db_reconciliation_without_row_id_keeps_legacy_dedup(): + import api.models as models + + row = _state_db_identity_message() + reconciled = models.reconciled_state_db_messages_for_session( + _state_db_reconciliation_session([]), + state_messages=[row, dict(row)], + ) + + assert reconciled == [row] + + +def test_missing_sidecar_recovery_preserves_identical_state_rows_by_row_identity( + monkeypatch, + tmp_path, +): + """Materialized state.db rows remain distinct through load/save repair.""" + import api.config as config + import api.models as models + from api.helpers import public_session_projection + from api.session_recovery import recover_missing_sidecars_from_state_db + + sid = "missing_sidecar_identical_state_rows" + session_dir = tmp_path / "sessions" + session_dir.mkdir() + state_db = tmp_path / "state.db" + _make_state_db( + state_db, + sid, + [ + {"role": "assistant", "content": "same", "timestamp": 1000.0}, + {"role": "assistant", "content": "same", "timestamp": 1000.0}, + ], + ) + monkeypatch.setattr(config, "SESSION_DIR", session_dir, raising=False) + monkeypatch.setattr(config, "SESSION_INDEX_FILE", session_dir / "_index.json", raising=False) + monkeypatch.setattr(models, "SESSION_DIR", session_dir, raising=False) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", session_dir / "_index.json", raising=False) + monkeypatch.setattr(models, "SESSIONS", OrderedDict(), raising=False) + + recovered = recover_missing_sidecars_from_state_db( + session_dir=session_dir, + state_db_path=state_db, + ) + assert recovered["materialized"] == 1 + assert recovered["details"] == [ + {"session_id": sid, "materialized": True, "messages": 2} + ] + + materialized = json.loads((session_dir / f"{sid}.json").read_text(encoding="utf-8")) + assert [message["_state_db_row_id"] for message in materialized["messages"]] == [1, 2] + + loaded = models.Session.load(sid) + assert loaded is not None + assert len(loaded.messages) == 2 + loaded.save(skip_index=True) + reloaded = models.Session.load(sid) + assert reloaded is not None + assert [message["_state_db_row_id"] for message in reloaded.messages] == [1, 2] + + public = public_session_projection({"messages": reloaded.messages}) + assert all("_state_db_row_id" not in message for message in public["messages"]) + + def test_sidebar_state_db_overlay_preserves_numeric_actual_count(): import api.models as models @@ -1687,8 +1876,11 @@ def test_state_db_reconciliation_preserves_repeated_sidecar_rows(monkeypatch, tm routes.handle_get(handler, urlparse(handler.path)) assert handler.status == 200 messages = handler.response_json["session"]["messages"] - assert [m["content"] for m in messages] == ["", "", "done"] - assert handler.response_json["session"]["message_count"] == 3 + # The two sidecar rows are byte-for-byte identical strict JSON payloads, so + # session load repairs them as one replay. Distinct timestamps, ids, model + # provenance, request ids, or other durable payload fields remain separate. + assert [m["content"] for m in messages] == ["", "done"] + assert handler.response_json["session"]["message_count"] == 2 def test_cancelled_partial_sidecar_owns_display_over_state_db_replay(monkeypatch, tmp_path): From da7fc6eeb409878d7fa467ac5e9e4a90f528b5e7 Mon Sep 17 00:00:00 2001 From: Anthony Ruiz Date: Sun, 27 Sep 2026 04:48:26 +0000 Subject: [PATCH 2/5] test(webui): cover sanitized boundary replay --- ...issue7032_context_replay_token_identity.py | 82 ++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/tests/test_issue7032_context_replay_token_identity.py b/tests/test_issue7032_context_replay_token_identity.py index 92d5c9b3d40..ad20d12ddd2 100644 --- a/tests/test_issue7032_context_replay_token_identity.py +++ b/tests/test_issue7032_context_replay_token_identity.py @@ -44,7 +44,7 @@ def _persisted_turn_one(*, with_api_content): def _settle_repeated_prompt_turn(previous_context, result_messages, *, authoritative): - from api.streaming import _settle_result_messages + from api.streaming import _resolve_active_turn_authority, _settle_result_messages previous_display = copy.deepcopy(previous_context) session = SimpleNamespace( @@ -59,9 +59,17 @@ def _settle_repeated_prompt_turn(previous_context, result_messages, *, authorita "timestamp": 200.0, "source": "webui", "attachments": [], - "current_turn_user_idx": len(previous_context) if authoritative else None, - "turn_id": "turn:2" if authoritative else "", + "current_turn_user_idx": None, + "turn_id": "", } + if authoritative: + identity = _resolve_active_turn_authority( + identity, + result={ + "current_turn_user_idx": len(previous_context), + "turn_id": "turn:2", + }, + ) _settle_result_messages( session, copy.deepcopy(previous_display), @@ -188,3 +196,71 @@ def test_current_answer_survives_full_history_replay_without_user_echo(): "display/context divergence: " f"display={display_answers} context={context_answers}" ) + + +def _sanitized_metadata_history(): + from api.streaming import _sanitize_messages_for_agent + + previous_context = _persisted_turn_one(with_api_content=False) + previous_context[0]["attachments"] = [ + {"name": "report.txt", "path": "/workspace/report.txt"} + ] + previous_context[0]["source"] = "webui" + previous_context[1]["reasoning_content"] = "private chain of thought" + agent_bound_history = _sanitize_messages_for_agent( + previous_context, + cfg={"webui": {"reasoning_content_replay": "strip"}}, + effective_model="test-model", + effective_provider="openai", + ) + assert "attachments" not in agent_bound_history[0] + assert "source" not in agent_bound_history[0] + assert "reasoning_content" not in agent_bound_history[1] + return previous_context, agent_bound_history + + +def _assert_exact_two_turn_projections(session): + expected_roles = ["user", "assistant", "user", "assistant"] + assert [message["role"] for message in session.messages] == expected_roles + assert [message["role"] for message in session.context_messages] == expected_roles + assert _assistant_answers(session.messages) == [ + "the report says A", + "the report now says B", + ] + assert _assistant_answers(session.context_messages) == [ + "the report says A", + "the report now says B", + ] + + +def test_authoritative_agent_projection_replay_with_user_echo_is_not_duplicated(): + """The exact Agent-bound prefix is history even after metadata sanitization.""" + previous_context, agent_bound_history = _sanitized_metadata_history() + result_messages = copy.deepcopy(agent_bound_history) + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": "the report now says B"}, + ] + + session = _settle_repeated_prompt_turn( + previous_context, + result_messages, + authoritative=True, + ) + + _assert_exact_two_turn_projections(session) + + +def test_authoritative_agent_projection_replay_without_user_echo_is_not_duplicated(): + """Out-of-band current users retain one durable history and one new answer.""" + previous_context, agent_bound_history = _sanitized_metadata_history() + result_messages = copy.deepcopy(agent_bound_history) + [ + {"role": "assistant", "content": "the report now says B"}, + ] + + session = _settle_repeated_prompt_turn( + previous_context, + result_messages, + authoritative=True, + ) + + _assert_exact_two_turn_projections(session) From fcbe0c2f1b7e3134849a4314b114dfbced527f69 Mon Sep 17 00:00:00 2001 From: Anthony Ruiz Date: Sun, 27 Sep 2026 05:10:37 +0000 Subject: [PATCH 3/5] fix(webui): reconcile sanitized replay boundaries --- api/models.py | 35 +++++ api/routes.py | 20 ++- api/streaming.py | 135 ++++++++++++++---- ...issue7032_context_replay_token_identity.py | 21 ++- .../test_native_image_turn_display_context.py | 45 ++++++ 5 files changed, 222 insertions(+), 34 deletions(-) diff --git a/api/models.py b/api/models.py index eae0966d204..1b52a387fd6 100644 --- a/api/models.py +++ b/api/models.py @@ -11858,6 +11858,19 @@ def _session_message_dedup_key(msg: dict): return key +def _invalid_state_db_row_payload_key(msg: dict): + """Key malformed-provenance rows by exact payload, never by bad aliases.""" + _, row_id_valid = _state_db_row_identity_details(msg) + if row_id_valid: + return None + without_row_identity = { + key: value + for key, value in msg.items() + if key not in {"_row_id", "_state_db_row_id", "_db_row_id", "state_db_row_id"} + } + return _session_message_dedup_key(without_row_identity) + + def _normalized_session_message_content(msg: dict): """Visible identity for a message's content. @@ -12873,6 +12886,28 @@ def _state_row_is_truncated( content_key = _cached_message_key(msg, "content_state") if preserve_native_image_row: row_id, row_id_valid = _state_db_row_identity_details(msg) + if not row_id_valid: + # The native-image bridge already proved this scalar is a + # payload-exact mirror candidate. Invalid provenance cannot + # prove another durable row, but the ordinary fail-closed key + # uses object identity and would append the same malformed row + # again on every reconciliation pass. Collapse only against an + # earlier malformed mirror with the exact payload key; valid + # row identities and conflicting provider bytes remain distinct. + invalid_payload_key = _invalid_state_db_row_payload_key(msg) + invalid_payload_duplicate = next( + ( + candidate + for candidate in merged_messages + if isinstance(candidate, dict) + and _invalid_state_db_row_payload_key(candidate) + == invalid_payload_key + ), + None, + ) + if invalid_payload_duplicate is not None: + _merge_session_display_metadata(invalid_payload_duplicate, msg) + continue existing = ( merged_by_row_id.get(row_id) if row_id_valid and row_id is not None diff --git a/api/routes.py b/api/routes.py index 444f61fdcf2..77a801365ba 100644 --- a/api/routes.py +++ b/api/routes.py @@ -25418,6 +25418,7 @@ def _handle_chat_sync(handler, body): ) from api.streaming import ( _WEBUI_PROGRESS_PROMPT, + _record_agent_history_replay_authority, _resolve_active_turn_authority, _sanitize_messages_for_agent, _compact_session_image_parts_for_persistence, @@ -25463,17 +25464,22 @@ def _handle_chat_sync(handler, body): "current_turn_user_idx": None, "turn_id": "", } + _sync_agent_bound_history = _sanitize_messages_for_agent( + _previous_context_messages, + cfg=get_config(), + effective_model=_model, + effective_provider=_provider, + effective_base_url=_base_url, + ) + _record_agent_history_replay_authority( + _sync_active_turn_identity, + _sync_agent_bound_history, + ) result = agent.run_conversation( user_message=workspace_ctx + msg, system_message=workspace_system_msg, - conversation_history=_sanitize_messages_for_agent( - _previous_context_messages, - cfg=get_config(), - effective_model=_model, - effective_provider=_provider, - effective_base_url=_base_url, - ), + conversation_history=_sync_agent_bound_history, task_id=s.session_id, persist_user_message=msg, ) diff --git a/api/streaming.py b/api/streaming.py index c1255d8f627..67512a83b2f 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -2079,6 +2079,67 @@ def _active_turn_boundary_is_valid(identity): ) +_AGENT_HISTORY_REPLAY_DIGESTS_FIELD = 'agent_history_replay_digests' + + +def _record_agent_history_replay_authority(identity, agent_bound_history): + """Bind settlement to the exact history projection supplied to the Agent.""" + if not isinstance(identity, dict): + return + identity.pop(_AGENT_HISTORY_REPLAY_DIGESTS_FIELD, None) + if not agent_bound_history: + return + digests = [] + for message in agent_bound_history: + digest = _canonical_replay_digest(message) + if digest is None: + return + digests.append(digest) + identity[_AGENT_HISTORY_REPLAY_DIGESTS_FIELD] = tuple(digests) + + +def _restore_agent_history_replay_prefix( + result_messages, + previous_context, + identity, +): + """Restore durable history when result replays the exact Agent input. + + The WebUI sanitizer deliberately removes persistence-only metadata before + provider replay. Comparing that projection to durable rows at settlement + therefore cannot be payload-equal. The request-local digest tuple records + the exact projection actually supplied to the Agent; only a complete, + position-for-position match may replace the replayed prefix with its durable + counterpart. Incomparable or differently sized projections fail closed. + """ + result_messages = list(result_messages or []) + previous_context = list(previous_context or []) + digests = ( + identity.get(_AGENT_HISTORY_REPLAY_DIGESTS_FIELD) + if isinstance(identity, dict) + else None + ) + if ( + type(digests) is not tuple + or len(digests) != len(previous_context) + or len(result_messages) < len(digests) + or ( + _active_turn_boundary_is_valid(identity) + and identity['current_turn_user_idx'] != len(previous_context) + ) + ): + return result_messages, False + for message, expected_digest in zip(result_messages, digests, strict=False): + if _canonical_replay_digest(message) != expected_digest: + return result_messages, False + history_size = len(digests) + return ( + copy.deepcopy(previous_context) + + copy.deepcopy(result_messages[history_size:]), + True, + ) + + def _exact_out_of_band_history_size(result_messages, previous_context): """Return ``len(previous_context)`` when the result is exact history + delta. @@ -2924,6 +2985,13 @@ def _settle_result_messages( source, active_turn_identity, ): + result_messages, agent_history_replay_prefix = ( + _restore_agent_history_replay_prefix( + result_messages, + previous_context_messages, + active_turn_identity, + ) + ) result_messages, repeated_exact_history_prefix = ( _collapse_repeated_exact_history_prefixes( previous_context_messages, @@ -2932,6 +3000,8 @@ def _settle_result_messages( ) ) result_has_authoritative_full_history_prefix = ( + agent_history_replay_prefix + or repeated_exact_history_prefix or _result_has_authoritative_full_history_prefix( result_messages, @@ -3050,7 +3120,7 @@ def _settle_result_messages( ): history_size = len(previous_context_messages) session.context_messages = ( - copy.deepcopy(previous_context_messages) + list(copy.deepcopy(previous_context_messages)) + _deduplicate_context_messages(next_context_messages[history_size:]) ) else: @@ -12896,18 +12966,23 @@ def _periodic_checkpoint(): _register_pending_user_timestamp_identity( agent.run_conversation, s, _persist_user_timestamp ) + _agent_bound_history = _sanitize_messages_for_agent( + _previous_context_messages, + cfg=_cfg, + effective_model=resolved_model, + effective_provider=resolved_provider, + effective_base_url=resolved_base_url, + requested_provider=(_session_requested_provider or ""), + ) + _record_agent_history_replay_authority( + _active_turn_identity, + _agent_bound_history, + ) _run_conversation_kwargs = _build_run_conversation_kwargs( agent.run_conversation, user_message=user_message, system_message=workspace_system_msg, - conversation_history=_sanitize_messages_for_agent( - _previous_context_messages, - cfg=_cfg, - effective_model=resolved_model, - effective_provider=resolved_provider, - effective_base_url=resolved_base_url, - requested_provider=(_session_requested_provider or ""), - ), + conversation_history=_agent_bound_history, conversation_history_revision=_conversation_history_revision, task_id=session_id, persist_user_message=msg_text, @@ -13489,18 +13564,23 @@ def _periodic_checkpoint(): s, _heal_persist_user_timestamp, ) + _heal_agent_bound_history = _sanitize_messages_for_agent( + _heal_context_messages, + cfg=_cfg, + effective_model=resolved_model, + effective_provider=resolved_provider, + effective_base_url=resolved_base_url, + requested_provider=(_session_requested_provider or ""), + ) + _record_agent_history_replay_authority( + _active_turn_identity, + _heal_agent_bound_history, + ) _heal_kwargs = _build_run_conversation_kwargs( agent.run_conversation, user_message=user_message, system_message=workspace_system_msg, - conversation_history=_sanitize_messages_for_agent( - _heal_context_messages, - cfg=_cfg, - effective_model=resolved_model, - effective_provider=resolved_provider, - effective_base_url=resolved_base_url, - requested_provider=(_session_requested_provider or ""), - ), + conversation_history=_heal_agent_bound_history, conversation_history_revision=( _heal_conversation_history_revision ), @@ -14856,18 +14936,23 @@ def _periodic_checkpoint(): s, _heal_persist_user_timestamp, ) + _heal_agent_bound_history = _sanitize_messages_for_agent( + _heal_context_messages, + cfg=_cfg, + effective_model=resolved_model, + effective_provider=resolved_provider, + effective_base_url=resolved_base_url, + requested_provider=(_session_requested_provider or ""), + ) + _record_agent_history_replay_authority( + _active_turn_identity, + _heal_agent_bound_history, + ) _heal_kwargs2 = _build_run_conversation_kwargs( _heal_agent.run_conversation, user_message=user_message, system_message=workspace_system_msg, - conversation_history=_sanitize_messages_for_agent( - _heal_context_messages, - cfg=_cfg, - effective_model=resolved_model, - effective_provider=resolved_provider, - effective_base_url=resolved_base_url, - requested_provider=(_session_requested_provider or ""), - ), + conversation_history=_heal_agent_bound_history, conversation_history_revision=( _heal_conversation_history_revision ), diff --git a/tests/test_issue7032_context_replay_token_identity.py b/tests/test_issue7032_context_replay_token_identity.py index ad20d12ddd2..4f206c100c1 100644 --- a/tests/test_issue7032_context_replay_token_identity.py +++ b/tests/test_issue7032_context_replay_token_identity.py @@ -43,8 +43,18 @@ def _persisted_turn_one(*, with_api_content): return [user, assistant] -def _settle_repeated_prompt_turn(previous_context, result_messages, *, authoritative): - from api.streaming import _resolve_active_turn_authority, _settle_result_messages +def _settle_repeated_prompt_turn( + previous_context, + result_messages, + *, + authoritative, + agent_bound_history=None, +): + from api.streaming import ( + _canonical_replay_digest, + _resolve_active_turn_authority, + _settle_result_messages, + ) previous_display = copy.deepcopy(previous_context) session = SimpleNamespace( @@ -62,6 +72,11 @@ def _settle_repeated_prompt_turn(previous_context, result_messages, *, authorita "current_turn_user_idx": None, "turn_id": "", } + if agent_bound_history is not None: + identity["agent_history_replay_digests"] = tuple( + _canonical_replay_digest(message) + for message in agent_bound_history + ) if authoritative: identity = _resolve_active_turn_authority( identity, @@ -245,6 +260,7 @@ def test_authoritative_agent_projection_replay_with_user_echo_is_not_duplicated( previous_context, result_messages, authoritative=True, + agent_bound_history=agent_bound_history, ) _assert_exact_two_turn_projections(session) @@ -261,6 +277,7 @@ def test_authoritative_agent_projection_replay_without_user_echo_is_not_duplicat previous_context, result_messages, authoritative=True, + agent_bound_history=agent_bound_history, ) _assert_exact_two_turn_projections(session) diff --git a/tests/test_native_image_turn_display_context.py b/tests/test_native_image_turn_display_context.py index ae9205b1f63..15d1ceb2601 100644 --- a/tests/test_native_image_turn_display_context.py +++ b/tests/test_native_image_turn_display_context.py @@ -18,6 +18,7 @@ _new_turn_context_from_messages, _active_turn_authority, _find_active_turn_checkpoint_index, + _record_agent_history_replay_authority, _sanitize_messages_for_agent, _settle_result_messages, ) @@ -1288,6 +1289,7 @@ def test_settlement_reload_and_next_turn_keep_one_clean_bubble_and_rich_context( "turn_id": "agent-turn-second", "agent_turn_boundary_resolved": True, }) + _record_agent_history_replay_authority(second_identity, model_history) checkpoint = _materialize_active_turn_user(second_identity, second_text, "webui") second_identity["checkpoint"] = checkpoint previous_display = [*display_rows, checkpoint] @@ -2056,6 +2058,49 @@ def test_untrusted_native_image_row_identity_deduplicates_stably( session.context_messages = context +def test_untrusted_native_image_row_identity_preserves_payload_distinct_rows(): + import api.models as models + + timestamp = 881.0 + session, identity, api_content = _settle_image_turn( + timestamp=timestamp, + agent_row_id=41, + ) + context_user = next( + message for message in session.context_messages + if message.get("_active_turn_token") == identity["token"] + ) + mirror = _durable_agent_content(context_user["content"]) + state_rows = [ + { + "role": "user", + "content": mirror, + "timestamp": timestamp, + "api_content": payload, + "_state_db_row_id": "malformed", + } + for payload in (api_content, f"{api_content} ") + ] + + for _ in range(3): + context = models.reconciled_state_db_messages_for_session( + session, + prefer_context=True, + state_messages=state_rows, + ) + mirrored_rows = [ + message for message in context + if message.get("content") == mirror + and message.get("timestamp") == timestamp + ] + assert len(mirrored_rows) == 2 + assert {message["api_content"] for message in mirrored_rows} == { + api_content, + f"{api_content} ", + } + session.context_messages = context + + def test_unlinked_state_db_image_projection_uses_existing_reconciliation(): import api.models as models From 36dbd0b277bd25d6ad8d230716a702ff4bcdc277 Mon Sep 17 00:00:00 2001 From: Anthony Ruiz Date: Mon, 24 Aug 2026 22:40:12 +0000 Subject: [PATCH 4/5] perf(webui): skip replay repair for sidecars already proven clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replay repair runs on every session LOAD, not just on save, and it dominates cold-load cost. It re-serializes every assistant message to canonical JSON and SHA-256s it, for the session AND for every compaction ancestor walked by the lineage — so a chat with 4 ancestors pays it five times per tab refresh. A py-spy profile taken under 6 concurrent loads on a live deployment put `_canonical_message_digest` as the #1 self-time frame. Because the work happens under the GIL it does not overlap between tabs: one load took 0.55s while six concurrent loads took 10.83s for the slowest (19.6x). The work is also almost always for nothing. Sampling the 60 most recent sidecars, repair changed nothing in 60 of 60 cases — 11.8s of pure waste. A file only needs repairing once; a clean file stays clean until it is rewritten. Memoize the NEGATIVE verdict only, keyed by the sha256 of the exact file bytes (already computed by the primary load path for revision tracking). A hit means "these exact bytes were proven to need no repair" and skips the pipeline. Any write changes the digest and therefore misses the cache. Measured on 6 real sidecars, same machine, warm: single load (median) 136ms -> 46ms (2.9x) single load (6 sessions) 875ms -> 302ms (2.9x) 6 concurrent loads (wall) 1054ms -> 279ms (3.8x) 6 concurrent loads (slowest) 1053ms -> 277ms (3.8x) Safety. Skipping repair is equivalent to running it because the collapse helpers never mutate their input: they build new lists and return (result, changed), returning the input untouched when changed is False. A test pins that property, so a future helper that starts mutating in place fails the suite instead of silently serving unrepaired sessions. A positive verdict is deliberately not cached: a file needing repair gets rewritten, so caching it would key on bytes that no longer exist. The cache is bounded (512 entries, LRU) and a missing digest forces the full pipeline. --- api/models.py | 89 +++++++++++- tests/test_replay_repair_clean_cache.py | 184 ++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 tests/test_replay_repair_clean_cache.py diff --git a/api/models.py b/api/models.py index 1b52a387fd6..ceace5d5485 100644 --- a/api/models.py +++ b/api/models.py @@ -1270,10 +1270,15 @@ def _read_metadata_json_prefix(path, max_prefix_bytes=_METADATA_PREFIX_MAX_BYTES def _load_session_from_path(path: Path) -> "Session | None": """Load a session from an explicit JSON path without consulting SESSION_DIR.""" try: - data = json.loads(path.read_text(encoding='utf-8')) + raw = path.read_bytes() + data = json.loads(raw) except Exception: return None - data, _, _ = _repair_session_message_projections(data) + # sha256 over the raw bytes is cheap next to the repair pipeline it lets us + # skip (~10ms vs ~485ms on a large sidecar). + data, _, _ = _repair_session_message_projections_cached( + data, hashlib.sha256(raw).hexdigest() + ) return Session(**data) @@ -1871,8 +1876,13 @@ def load(cls, sid): # cache write is only committed if the file didn't change under us # during the parse (TOCTOU guard against an atomic replace mid-read). _pre_read_sig = _sidecar_stat_signature(p) - data = json.loads(p.read_text(encoding='utf-8')) - data, messages_changed, context_changed = _repair_session_message_projections(data) + raw = p.read_bytes() + data = json.loads(raw) + data, messages_changed, context_changed = ( + _repair_session_message_projections_cached( + data, hashlib.sha256(raw).hexdigest() + ) + ) session = cls(**data) if messages_changed or context_changed: try: @@ -3248,6 +3258,77 @@ def _repair_session_message_projections(data: dict) -> tuple[dict, bool, bool]: return data, messages_changed, context_changed +# Replay repair runs on every session LOAD, not just on save, and it is the +# single most expensive step of a cold load: it re-serializes every assistant +# message to canonical JSON and SHA-256s it, for the session AND for every +# compaction ancestor walked by the lineage. +# +# Measured on a live deployment: 485ms of repair for 171ms of actual JSON +# reading (74% of load cost), with _canonical_message_digest the #1 self-time +# frame in a py-spy profile taken under 6 concurrent loads. Because the work +# happens under the GIL it does not overlap between tabs: 1 load took 0.55s, +# 6 concurrent loads took 10.83s for the slowest (19.6x). +# +# Crucially the work is almost always for nothing: sampling the 60 most recent +# sidecars, repair changed NOTHING in 60/60 cases — 11.8s of pure waste. A file +# only needs repairing once; a clean file stays clean until it is rewritten. +# +# So we memoize the NEGATIVE verdict only, keyed by the exact bytes of the file +# (sha256). A hit means "these exact bytes were already proven to need no +# repair", which lets us skip the whole pipeline. Anything that changes the file +# changes the digest and therefore misses the cache. +# +# This is safe because the collapse helpers never mutate messages in place: they +# build new lists and return (result, changed). When changed is False the input +# is returned untouched, so skipping the call yields identical data. That +# property is pinned by tests; if a future collapse helper starts mutating its +# input, those tests fail rather than silently serving unrepaired sessions. +# +# A positive verdict is deliberately NOT cached: a file that needs repair gets +# rewritten, so caching "needs repair" would key on bytes that no longer exist. +_REPAIR_CLEAN_CACHE_SIZE = 512 +_repair_clean_digests: "collections.OrderedDict[str, bool]" = collections.OrderedDict() +_repair_clean_lock = threading.Lock() + + +def _repair_clean_cache_lookup(digest: str | None) -> bool: + """True when these exact file bytes were already proven repair-free.""" + if not digest: + return False + with _repair_clean_lock: + if digest in _repair_clean_digests: + _repair_clean_digests.move_to_end(digest) + return True + return False + + +def _repair_clean_cache_store(digest: str | None) -> None: + """Record that these exact file bytes need no repair.""" + if not digest: + return + with _repair_clean_lock: + _repair_clean_digests[digest] = True + _repair_clean_digests.move_to_end(digest) + while len(_repair_clean_digests) > _REPAIR_CLEAN_CACHE_SIZE: + _repair_clean_digests.popitem(last=False) + + +def _repair_session_message_projections_cached( + data: dict, digest: str | None +) -> tuple[dict, bool, bool]: + """``_repair_session_message_projections`` skipped for known-clean bytes. + + ``digest`` must identify the exact file content ``data`` was parsed from. + Pass ``None`` to force the full pipeline. + """ + if _repair_clean_cache_lookup(digest): + return data, False, False + data, messages_changed, context_changed = _repair_session_message_projections(data) + if not messages_changed and not context_changed: + _repair_clean_cache_store(digest) + return data, messages_changed, context_changed + + def _find_existing_assistant_for_journal_content( session, content: str, diff --git a/tests/test_replay_repair_clean_cache.py b/tests/test_replay_repair_clean_cache.py new file mode 100644 index 00000000000..e504ccff1bc --- /dev/null +++ b/tests/test_replay_repair_clean_cache.py @@ -0,0 +1,184 @@ +"""Regression tests for the known-clean replay-repair cache. + +Replay repair runs on every session LOAD and dominates cold-load cost (measured +485ms of repair for 171ms of JSON reading on a real deployment). It is also +almost always a no-op: on 60 consecutive real sidecars it changed nothing 60 +times. `_repair_session_message_projections_cached` memoizes only that negative +verdict, keyed by the exact file bytes. + +These tests pin the three properties that make the cache safe: + +1. a cache hit returns data identical to running the full pipeline; +2. a session that genuinely needs repair is ALWAYS repaired, never cached away; +3. the collapse helpers do not mutate their input, which is what makes skipping + them equivalent to running them. +""" + +import copy +import hashlib +import json + +import pytest + +from api import models + + +@pytest.fixture(autouse=True) +def _clear_cache(): + models._repair_clean_digests.clear() + yield + models._repair_clean_digests.clear() + + +def _clean_session(session_id="20260101_000000_clean"): + return { + "session_id": session_id, + "messages": [ + {"role": "user", "content": "bonjour"}, + {"role": "assistant", "content": "salut", "id": "a1"}, + {"role": "user", "content": "merci"}, + {"role": "assistant", "content": "de rien", "id": "a2"}, + ], + } + + +def _dirty_session(session_id="20260101_000000_dirty"): + """Two adjacent assistant rows with identical payloads: a real replay.""" + duplicated = {"role": "assistant", "content": "reponse", "id": "dup"} + return { + "session_id": session_id, + "messages": [ + {"role": "user", "content": "question"}, + dict(duplicated), + dict(duplicated), + ], + } + + +def test_cache_hit_matches_uncached_result(): + """A cached load must return exactly what the full pipeline returns.""" + reference, ref_msg, ref_ctx = models._repair_session_message_projections( + _clean_session() + ) + + digest = "d" * 64 + first, msg1, ctx1 = models._repair_session_message_projections_cached( + _clean_session(), digest + ) + assert (msg1, ctx1) == (ref_msg, ref_ctx) + assert first == reference + + # Second call takes the cache path. + assert digest in models._repair_clean_digests + second, msg2, ctx2 = models._repair_session_message_projections_cached( + _clean_session(), digest + ) + assert (msg2, ctx2) == (False, False) + assert second == reference + + +def test_session_needing_repair_is_never_cached(): + """A positive verdict must not be memoized, and must repair every time.""" + digest = "e" * 64 + + _, msg1, _ = models._repair_session_message_projections_cached( + _dirty_session(), digest + ) + assert msg1 is True, "the duplicated assistant row should have been collapsed" + assert digest not in models._repair_clean_digests + + # Same bytes again: repair must run again, not be skipped. + repaired, msg2, _ = models._repair_session_message_projections_cached( + _dirty_session(), digest + ) + assert msg2 is True + assert len(repaired["messages"]) == 2 + + +def test_distinct_digests_do_not_share_verdicts(): + """A clean file must never authorize skipping repair on a different file.""" + models._repair_session_message_projections_cached(_clean_session(), "a" * 64) + + repaired, changed, _ = models._repair_session_message_projections_cached( + _dirty_session(), "b" * 64 + ) + assert changed is True + assert len(repaired["messages"]) == 2 + + +def test_missing_digest_forces_full_pipeline(): + """Without a trustworthy key the cache must not engage.""" + _, changed, _ = models._repair_session_message_projections_cached( + _clean_session(), None + ) + assert changed is False + assert not models._repair_clean_digests + + repaired, changed, _ = models._repair_session_message_projections_cached( + _dirty_session(), None + ) + assert changed is True + assert len(repaired["messages"]) == 2 + + +def test_collapse_helpers_do_not_mutate_input(): + """The safety hypothesis behind the cache. + + Skipping repair is only equivalent to running it because the collapse + helpers build new lists instead of editing messages in place. If that ever + changes, this test fails instead of the cache silently serving stale data. + """ + for factory in (_clean_session, _dirty_session): + messages = factory()["messages"] + snapshot = copy.deepcopy(messages) + models._collapse_replayed_assistant_rows(messages) + assert messages == snapshot, f"{factory.__name__}: input was mutated in place" + + +def test_cache_is_bounded(): + """The cache must not grow without limit on a long-lived server.""" + for i in range(models._REPAIR_CLEAN_CACHE_SIZE + 50): + models._repair_session_message_projections_cached( + _clean_session(), f"{i:064x}" + ) + assert len(models._repair_clean_digests) <= models._REPAIR_CLEAN_CACHE_SIZE + + +def test_load_path_repairs_dirty_session_then_caches_clean(tmp_path, monkeypatch): + """End-to-end through the real file loader.""" + monkeypatch.setattr(models, "SESSION_DIR", tmp_path) + + sid = "20260101_000000_dirty" + path = tmp_path / f"{sid}.json" + path.write_text(json.dumps(_dirty_session(sid)), encoding="utf-8") + + session = models._load_session_from_path(path) + assert session is not None + assert len(session.messages) == 2, "repair must run on a dirty sidecar" + + # A clean sidecar gets its digest memoized by the same loader. + clean_sid = "20260101_000000_clean" + clean_path = tmp_path / f"{clean_sid}.json" + raw = json.dumps(_clean_session(clean_sid)).encode("utf-8") + clean_path.write_bytes(raw) + + loaded = models._load_session_from_path(clean_path) + assert loaded is not None + assert len(loaded.messages) == 4 + assert hashlib.sha256(raw).hexdigest() in models._repair_clean_digests + + +def test_rewritten_file_invalidates_the_verdict(tmp_path, monkeypatch): + """New bytes must never inherit the previous verdict.""" + monkeypatch.setattr(models, "SESSION_DIR", tmp_path) + + sid = "20260101_000000_evolving" + path = tmp_path / f"{sid}.json" + path.write_bytes(json.dumps(_clean_session(sid)).encode("utf-8")) + assert models._load_session_from_path(path) is not None + + # The session is rewritten and now contains a replay. + path.write_bytes(json.dumps(_dirty_session(sid)).encode("utf-8")) + session = models._load_session_from_path(path) + assert session is not None + assert len(session.messages) == 2, "changed bytes must miss the cache" From 8e6805157843b14cdd18dbf761f566fdd1a053fb Mon Sep 17 00:00:00 2001 From: Anthony Ruiz Date: Wed, 23 Sep 2026 09:54:18 +0000 Subject: [PATCH 5/5] test(webui): characterize overlapping cold replay-repair loads --- tests/test_replay_repair_clean_cache.py | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_replay_repair_clean_cache.py b/tests/test_replay_repair_clean_cache.py index e504ccff1bc..3b731a8097f 100644 --- a/tests/test_replay_repair_clean_cache.py +++ b/tests/test_replay_repair_clean_cache.py @@ -17,6 +17,8 @@ import copy import hashlib import json +import threading +from concurrent.futures import ThreadPoolExecutor import pytest @@ -168,6 +170,41 @@ def test_load_path_repairs_dirty_session_then_caches_clean(tmp_path, monkeypatch assert hashlib.sha256(raw).hexdigest() in models._repair_clean_digests +def test_overlapping_cold_loads_both_repair_before_warm_cache(tmp_path, monkeypatch): + """Cold joiners are safe, but this cache does not coalesce their work.""" + monkeypatch.setattr(models, "SESSION_DIR", tmp_path) + sid = "20260101_000000_cold_parallel" + path = tmp_path / f"{sid}.json" + raw = json.dumps(_clean_session(sid)).encode("utf-8") + path.write_bytes(raw) + + both_repairing = threading.Barrier(2, timeout=10) + lock = threading.Lock() + calls = [] + repair = models._repair_session_message_projections + + def overlapping_repair(data): + with lock: + calls.append(data["session_id"]) + both_repairing.wait() + return repair(data) + + monkeypatch.setattr(models, "_repair_session_message_projections", overlapping_repair) + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(models._load_session_from_path, path) for _ in range(2)] + loaded = [future.result(timeout=15) for future in futures] + + assert calls == [sid, sid] # Both miss before the first verdict is stored. + assert all( + session is not None and session.messages == _clean_session(sid)["messages"] + for session in loaded + ) + assert hashlib.sha256(raw).hexdigest() in models._repair_clean_digests + # Subsequent warm loads do skip the repair pipeline. + assert models._load_session_from_path(path) is not None + assert len(calls) == 2 + + def test_rewritten_file_invalidates_the_verdict(tmp_path, monkeypatch): """New bytes must never inherit the previous verdict.""" monkeypatch.setattr(models, "SESSION_DIR", tmp_path)