From f5da99e391dcc26327d4abb2aea2f2572863c690 Mon Sep 17 00:00:00 2001 From: David Bellamy <12414531+DavidBellamy@users.noreply.github.com> Date: Sun, 19 Apr 2026 04:41:58 +0000 Subject: [PATCH] fix(rollout/session): reseed session instead of erroring when no assistant persisted When the agent retries with a divergent user message before the first assistant turn has been persisted, _try_detect_and_rollback_to_assistant _checkpoint previously raised MessageValidationError because the matched prefix contained no assistant checkpoint to roll back to. This killed trials that would otherwise recover on retry. Observed repro: the first LLM turn of terminus-2 hit max_tokens=2048 truncation, errored out before the assistant response could be stored, and the agent retried with a modified user prompt. Stored=[sys,user], request=[sys,user_modified,...], match_len=1, no assistant in stored[:1] -> raise. battery-charging-optimization trials 6 and 7 on RL360 job 1565412 hit this path and failed outright rather than just returning reward=0. Fix: in the checkpoint_index < 0 branch, if num_assistant == 0 the stored state is a prompt-only seed whose first assistant turn never persisted. Clear the session and return True so the caller treats this as a fresh session. The existing raise path is preserved for cases where at least one assistant has been stored (caller is trying to jump back across an assistant turn, which remains disallowed). The caller (prepare_pretokenized) now early-returns None on the reseed signal, equivalent to the existing empty-session branch. --- miles/rollout/session/linear_trajectory.py | 48 ++++++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/miles/rollout/session/linear_trajectory.py b/miles/rollout/session/linear_trajectory.py index 79d092349b2..3db681bf5c7 100644 --- a/miles/rollout/session/linear_trajectory.py +++ b/miles/rollout/session/linear_trajectory.py @@ -77,7 +77,11 @@ def prepare_pretokenized( return None # 1. Detect agent retries and roll back (at most one assistant step). - self._try_detect_and_rollback_to_assistant_checkpoint(request_messages) + # Returns True iff the session was reseeded (see below); in that case + # there is no stored history to merge against, so fall through to the + # same None-return as an empty session. + if self._try_detect_and_rollback_to_assistant_checkpoint(request_messages): + return None # 2. Confirm the (possibly rolled-back) stored messages are a prefix of request, # and that each appended message role is in tito_tokenizer.allowed_append_roles. try: @@ -143,9 +147,16 @@ def update_pretokenized_state( def _try_detect_and_rollback_to_assistant_checkpoint( self, request_messages: list[dict[str, Any]], - ) -> None: + ) -> bool: """Detect if *request_messages* diverges from stored history and roll back. + Returns ``True`` iff the session was reseeded to empty because the + matched prefix contains no assistant and no assistant turn has been + persisted yet (see "pre-assistant re-seed" below). Returns ``False`` + for every other outcome (no rollback, or a normal rollback to a + prior assistant checkpoint). Callers should treat a ``True`` return + as equivalent to entering a fresh session. + In agentic workflows the agent may retry from an earlier point — for example, re-running a tool call with different arguments. When that happens the new request shares a common prefix with the stored messages @@ -185,10 +196,21 @@ def _try_detect_and_rollback_to_assistant_checkpoint( - The stored history is empty. - *request_messages* is a strict extension of stored messages (``match_len >= len(stored)``). + + Pre-assistant re-seed: if the matched prefix contains no assistant + AND no assistant turn has been persisted yet (``num_assistant == 0``), + the stored state is a prompt-only seed (system/user) from a prior + request whose first assistant turn never completed, e.g. the LLM hit + ``max_tokens`` before emitting a stop token, errored in flight, or + timed out before the session record could be written. In that case + the session is reseeded to empty and ``True`` is returned so the + caller takes the fresh-session branch. This replaces a previous + ``MessageValidationError`` that killed trials which would otherwise + have recovered on retry. """ stored = self.messages if not stored or not self.trajectory_token_ids: - return + return False match_len = 0 for i in range(min(len(request_messages), len(stored))): @@ -198,7 +220,7 @@ def _try_detect_and_rollback_to_assistant_checkpoint( break if match_len >= len(stored): - return + return False # Find the last assistant message within the matched prefix. rollback_msg_end = None @@ -211,6 +233,23 @@ def _try_detect_and_rollback_to_assistant_checkpoint( assistant_count += 1 if checkpoint_index < 0: + if self.num_assistant == 0: + # Pre-assistant re-seed: stored is a prompt-only state from + # a prior request whose first assistant turn never + # persisted. Clear state and let the caller treat this as a + # fresh session. + logger.info( + "Reseeding session: no assistant checkpoint stored yet, " + "request diverges at index %d (stored=%d msgs, request=%d msgs)", + match_len, + len(stored), + len(request_messages), + ) + self.messages = [] + self.trajectory_token_ids = [] + self.records = [] + self.num_assistant = 0 + return True raise MessageValidationError( f"rollback failed: no assistant message found in the first " f"{match_len} matched messages (stored has {len(stored)} messages, " @@ -240,6 +279,7 @@ def _try_detect_and_rollback_to_assistant_checkpoint( self.trajectory_token_ids = self.trajectory_token_ids[: checkpoint_index + 1] self.records = self.records[: checkpoint_index + 1] self.num_assistant = checkpoint_index + 1 + return False class SessionRegistry: