Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions miles/rollout/session/linear_trajectory.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,11 @@ def prepare_pretokenized(
return None

# 1. Detect agent retries and roll back (at most one assistant step).
self._try_detect_and_rollback_to_assistant_checkpoint(request_messages)
# Returns True iff the session was reseeded (see below); in that case
# there is no stored history to merge against, so fall through to the
# same None-return as an empty session.
if self._try_detect_and_rollback_to_assistant_checkpoint(request_messages):
return None
# 2. Confirm the (possibly rolled-back) stored messages are a prefix of request,
# and that each appended message role is in tito_tokenizer.allowed_append_roles.
try:
Expand Down Expand Up @@ -143,9 +147,16 @@ def update_pretokenized_state(
def _try_detect_and_rollback_to_assistant_checkpoint(
self,
request_messages: list[dict[str, Any]],
) -> None:
) -> bool:
"""Detect if *request_messages* diverges from stored history and roll back.

Returns ``True`` iff the session was reseeded to empty because the
matched prefix contains no assistant and no assistant turn has been
persisted yet (see "pre-assistant re-seed" below). Returns ``False``
for every other outcome (no rollback, or a normal rollback to a
prior assistant checkpoint). Callers should treat a ``True`` return
as equivalent to entering a fresh session.

In agentic workflows the agent may retry from an earlier point — for
example, re-running a tool call with different arguments. When that
happens the new request shares a common prefix with the stored messages
Expand Down Expand Up @@ -185,10 +196,21 @@ def _try_detect_and_rollback_to_assistant_checkpoint(
- The stored history is empty.
- *request_messages* is a strict extension of stored messages
(``match_len >= len(stored)``).

Pre-assistant re-seed: if the matched prefix contains no assistant
AND no assistant turn has been persisted yet (``num_assistant == 0``),
the stored state is a prompt-only seed (system/user) from a prior
request whose first assistant turn never completed, e.g. the LLM hit
``max_tokens`` before emitting a stop token, errored in flight, or
timed out before the session record could be written. In that case
the session is reseeded to empty and ``True`` is returned so the
caller takes the fresh-session branch. This replaces a previous
``MessageValidationError`` that killed trials which would otherwise
have recovered on retry.
"""
stored = self.messages
if not stored or not self.trajectory_token_ids:
return
return False

match_len = 0
for i in range(min(len(request_messages), len(stored))):
Expand All @@ -198,7 +220,7 @@ def _try_detect_and_rollback_to_assistant_checkpoint(
break

if match_len >= len(stored):
return
return False

# Find the last assistant message within the matched prefix.
rollback_msg_end = None
Expand All @@ -211,6 +233,23 @@ def _try_detect_and_rollback_to_assistant_checkpoint(
assistant_count += 1

if checkpoint_index < 0:
if self.num_assistant == 0:
# Pre-assistant re-seed: stored is a prompt-only state from
# a prior request whose first assistant turn never
# persisted. Clear state and let the caller treat this as a
# fresh session.
logger.info(
"Reseeding session: no assistant checkpoint stored yet, "
"request diverges at index %d (stored=%d msgs, request=%d msgs)",
match_len,
len(stored),
len(request_messages),
)
self.messages = []
self.trajectory_token_ids = []
self.records = []
self.num_assistant = 0
return True
Comment on lines +236 to +252

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The logic in this new branch appears to be unreachable from prepare_pretokenized given the current implementation:

  1. In prepare_pretokenized (line 76), the function returns None if self.token_ids is empty. Since self.token_ids is derived from self.trajectory_token_ids, and self.num_assistant is kept in sync with the length of self.trajectory_token_ids (see update_pretokenized_state at line 145), num_assistant == 0 implies token_ids is empty. Thus, prepare_pretokenized would have already returned None before calling this method.
  2. Even if called, _try_detect_and_rollback_to_assistant_checkpoint returns early at line 213 if self.trajectory_token_ids is empty (which is the case when num_assistant == 0).

If the intention is to handle reseeding when a divergence occurs before the first assistant turn is persisted, the check at line 76 in prepare_pretokenized (and line 212 here) might need to be adjusted. Additionally, when implementing logic to reset or rollback session state, ensure all generated outputs and metadata fields are cleared to their default state to prevent carrying over stale data from previous attempts.

References
  1. When implementing a function to reset objects for retry, ensure all generated outputs and metadata fields are cleared to their default state to prevent carrying over stale data from previous attempts.

raise MessageValidationError(
f"rollback failed: no assistant message found in the first "
f"{match_len} matched messages (stored has {len(stored)} messages, "
Expand Down Expand Up @@ -240,6 +279,7 @@ def _try_detect_and_rollback_to_assistant_checkpoint(
self.trajectory_token_ids = self.trajectory_token_ids[: checkpoint_index + 1]
self.records = self.records[: checkpoint_index + 1]
self.num_assistant = checkpoint_index + 1
return False


class SessionRegistry:
Expand Down