diff --git a/openhands-sdk/openhands/sdk/agent/agent.py b/openhands-sdk/openhands/sdk/agent/agent.py index a9dd301a17..1f860b9ff8 100644 --- a/openhands-sdk/openhands/sdk/agent/agent.py +++ b/openhands-sdk/openhands/sdk/agent/agent.py @@ -226,6 +226,21 @@ def _should_handle_non_multimodal_image_input( return _latest_user_message_contains_image(messages) and not llm.vision_is_active() +def _mask_condensation( + condensation: Condensation, conversation: LocalConversation +) -> Condensation: + """Mask registered secrets in a condensation's summary before it is emitted. + + The summary is durable model output, replayed to the agent on every later + step, and the summarizer sees each forgotten action's arguments verbatim -- + a token typed into a git URL included. Masked like any agent message (#4783). + """ + if not condensation.summary: + return condensation + mask = conversation.state.secret_registry.mask_secrets_in_output + return condensation.model_copy(update={"summary": mask(condensation.summary)}) + + @dataclass(frozen=True, slots=True) class _ActionBatch: """Immutable result of preparing a batch of actions for execution. @@ -757,7 +772,7 @@ def _step( # Process condensation event before agent sampels another action if isinstance(_messages_or_condensation, Condensation): - on_event(_messages_or_condensation) + on_event(_mask_condensation(_messages_or_condensation, conversation)) return _messages = _messages_or_condensation @@ -985,7 +1000,7 @@ async def _astep( ) if isinstance(_messages_or_condensation, Condensation): - on_event(_messages_or_condensation) + on_event(_mask_condensation(_messages_or_condensation, conversation)) return _messages = _messages_or_condensation diff --git a/openhands-sdk/openhands/sdk/context/condenser/llm_summarizing_condenser.py b/openhands-sdk/openhands/sdk/context/condenser/llm_summarizing_condenser.py index cc64d1e2b7..5e28f15bb1 100644 --- a/openhands-sdk/openhands/sdk/context/condenser/llm_summarizing_condenser.py +++ b/openhands-sdk/openhands/sdk/context/condenser/llm_summarizing_condenser.py @@ -1,7 +1,7 @@ import os from collections.abc import Sequence from enum import Enum -from typing import Final +from typing import ClassVar, Final from pydantic import Field, model_validator @@ -13,16 +13,17 @@ from openhands.sdk.context.condenser.utils import ( get_suffix_length_for_token_reduction, get_total_token_count, + render_event_for_summary, ) from openhands.sdk.context.prompts import render_template from openhands.sdk.context.view import View from openhands.sdk.event.base import LLMConvertibleEvent -from openhands.sdk.event.condenser import Condensation +from openhands.sdk.event.condenser import Condensation, CondensationSummaryEvent from openhands.sdk.event.llm_convertible import SystemPromptEvent from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.llm.exceptions.classifier import is_context_window_exceeded from openhands.sdk.logger import get_logger from openhands.sdk.observability.laminar import observe -from openhands.sdk.utils import maybe_truncate logger = get_logger(__name__) @@ -78,6 +79,13 @@ class LLMSummarizingCondenser(RollingCondenser): are treated as errors. """ + max_event_length: ClassVar[int] = 10_000 + """Longest a single forgotten event may be when shown to the summarizer, in + characters; longer ones keep their head and tail. A previous summary is exempt: + it is the agent's only record of everything condensed before it. A class + constant, not a field, so the condenser's settings schema is unchanged. + """ + hard_context_reset_max_retries: int = Field(default=5, gt=0) """Number of attempts to perform hard context reset before raising an error.""" @@ -221,6 +229,129 @@ def _build_summary_messages(self, event_strings: Sequence[str]) -> list[Message] Message(role="user", content=[TextContent(text=events_prompt)]), ] + def _render_forgotten_events( + self, + forgotten_events: Sequence[LLMConvertibleEvent], + max_event_str_length: int | None = None, + ) -> list[str]: + """Render forgotten events for the summarizer, whole up to the length caps. + + Every event is capped at ``max_event_length``, except a previous summary. + ``max_event_str_length`` -- set only when the summarizer overflowed and the + input is being shrunk -- caps every event, the previous summary included. + """ + rendered = [] + for event in forgotten_events: + limits = [max_event_str_length] + if not isinstance(event, CondensationSummaryEvent): + limits.append(self.max_event_length) + caps = [limit for limit in limits if limit is not None] + rendered.append( + render_event_for_summary(event, min(caps) if caps else None) + ) + return rendered + + def _shrunk_event_length( + self, + events: Sequence[LLMConvertibleEvent], + max_event_str_length: int | None, + ) -> int: + """The next per-event cap after the summarizer call failed. + + Chosen so the rendered total -- not just the longest event -- falls to + ``hard_context_reset_context_scaling`` of what was just sent: scaling the + longest event alone leaves every shorter one untouched, so a prompt made + of many mid-sized events is re-sent almost unchanged. Returns 0 when no + positive cap gets there, i.e. there is nothing left to cut. + """ + lengths = [ + len(text) + for text in self._render_forgotten_events(events, max_event_str_length) + ] + target = sum(lengths) * self.hard_context_reset_context_scaling + # The largest cap whose capped total fits the target (capping never grows + # an event, so the total is monotonic in the cap). + low, high = 0, max(lengths) + while low < high: + cap = (low + high + 1) // 2 + if sum(min(length, cap) for length in lengths) <= target: + low = cap + else: + high = cap - 1 + return low + + def _summarize( + self, forgotten_events: Sequence[LLMConvertibleEvent], summary_offset: int + ) -> Condensation: + """Summarize the forgotten events whole; shrink them only on overflow. + + Whole events can exceed a summarizer whose window is smaller than the + agent's. Rather than fail -- and fall through to a hard context reset that + forgets everything -- retry with every event cut further each time. Any + other failure is raised at once. + """ + max_event_str_length: int | None = None + for _ in range(self.hard_context_reset_max_retries): + try: + return self._generate_condensation( + forgotten_events=forgotten_events, + summary_offset=summary_offset, + max_event_str_length=max_event_str_length, + ) + except NoCondensationAvailableException as e: + cause = e.__cause__ + if not isinstance(cause, Exception) or not is_context_window_exceeded( + cause + ): + raise + max_event_str_length = self._shrunk_event_length( + forgotten_events, max_event_str_length + ) + if max_event_str_length <= 0: + raise NoCondensationAvailableException( + "Summarizer context exceeded with nothing left to cut." + ) from e + logger.warning( + "Summarizer context exceeded; retrying with each event cut to " + f"{max_event_str_length} characters." + ) + raise NoCondensationAvailableException( + "Summarizer context exceeded at every event length tried." + ) + + async def _asummarize( + self, forgotten_events: Sequence[LLMConvertibleEvent], summary_offset: int + ) -> Condensation: + """Async variant of :meth:`_summarize`.""" + max_event_str_length: int | None = None + for _ in range(self.hard_context_reset_max_retries): + try: + return await self._agenerate_condensation( + forgotten_events=forgotten_events, + summary_offset=summary_offset, + max_event_str_length=max_event_str_length, + ) + except NoCondensationAvailableException as e: + cause = e.__cause__ + if not isinstance(cause, Exception) or not is_context_window_exceeded( + cause + ): + raise + max_event_str_length = self._shrunk_event_length( + forgotten_events, max_event_str_length + ) + if max_event_str_length <= 0: + raise NoCondensationAvailableException( + "Summarizer context exceeded with nothing left to cut." + ) from e + logger.warning( + "Summarizer context exceeded; retrying with each event cut to " + f"{max_event_str_length} characters." + ) + raise NoCondensationAvailableException( + "Summarizer context exceeded at every event length tried." + ) + def _generate_condensation( self, forgotten_events: Sequence[LLMConvertibleEvent], @@ -244,11 +375,9 @@ def _generate_condensation( """ assert len(forgotten_events) > 0, "No events to condense." - # Convert events to strings for the template - event_strings = [ - maybe_truncate(str(forgotten_event), truncate_after=max_event_str_length) - for forgotten_event in forgotten_events - ] + event_strings = self._render_forgotten_events( + forgotten_events, max_event_str_length + ) messages = self._build_summary_messages(event_strings) @@ -382,15 +511,10 @@ def hard_context_reset( max_event_str_length=max_event_str_length, ) except Exception as e: - # If we haven't set a max_event_str_length yet, set it as the largest - # event string length. - if max_event_str_length is None: - max_event_str_length = max(len(str(event)) for event in view.events) - - # Since the summarization failed, reduce the max_event_str_length by 20% - assert max_event_str_length is not None - max_event_str_length = int( - max_event_str_length * self.hard_context_reset_context_scaling + # Since the summarization failed, cut every event to 80% of the + # longest one as rendered -- not of its 500-character preview. + max_event_str_length = max( + 1, self._shrunk_event_length(forgotten_events, max_event_str_length) ) # Log the exception so we can track these failures @@ -433,10 +557,7 @@ def get_condensation( "threshold." ) - return self._generate_condensation( - forgotten_events=forgotten_events, - summary_offset=summary_offset, - ) + return self._summarize(forgotten_events, summary_offset) # ------------------------------------------------------------------ # Async variants @@ -451,10 +572,9 @@ async def _agenerate_condensation( """Async variant of :meth:`_generate_condensation`.""" assert len(forgotten_events) > 0, "No events to condense." - event_strings = [ - maybe_truncate(str(fe), truncate_after=max_event_str_length) - for fe in forgotten_events - ] + event_strings = self._render_forgotten_events( + forgotten_events, max_event_str_length + ) messages = self._build_summary_messages(event_strings) @@ -505,10 +625,7 @@ async def aget_condensation( "progress threshold." ) - return await self._agenerate_condensation( - forgotten_events=forgotten_events, - summary_offset=summary_offset, - ) + return await self._asummarize(forgotten_events, summary_offset) async def ahard_context_reset( self, @@ -533,11 +650,8 @@ async def ahard_context_reset( max_event_str_length=max_event_str_length, ) except Exception as e: - if max_event_str_length is None: - max_event_str_length = max(len(str(ev)) for ev in view.events) - assert max_event_str_length is not None - max_event_str_length = int( - max_event_str_length * self.hard_context_reset_context_scaling + max_event_str_length = max( + 1, self._shrunk_event_length(forgotten_events, max_event_str_length) ) logger.warning( f"Hard context reset summarization failed: {e}. " diff --git a/openhands-sdk/openhands/sdk/context/condenser/prompts/summarizing_system.j2 b/openhands-sdk/openhands/sdk/context/condenser/prompts/summarizing_system.j2 index f2cfa2159b..5340c28437 100644 --- a/openhands-sdk/openhands/sdk/context/condenser/prompts/summarizing_system.j2 +++ b/openhands-sdk/openhands/sdk/context/condenser/prompts/summarizing_system.j2 @@ -1,5 +1,7 @@ You are maintaining a context-aware state summary for an interactive agent. You will be given a list of events corresponding to actions taken by the agent, which will include previous summaries. +A PREVIOUS SUMMARY is the agent's only memory of the work before it. Carry every fact in it forward (files read and what they contain, commands run and their results, decisions, changes) unless a later event changes it, and never report as not done or not checked what it or an event shows was done. +Keep every request, answer and correction the user gave; quote short answers (ids, names, values) verbatim. If the events being summarized contain ANY task-tracking, you MUST include a TASK_TRACKING section to maintain continuity. When referencing tasks make sure to preserve exact task IDs and statuses. diff --git a/openhands-sdk/openhands/sdk/context/condenser/utils.py b/openhands-sdk/openhands/sdk/context/condenser/utils.py index 3882785b4c..c74b799fd2 100644 --- a/openhands-sdk/openhands/sdk/context/condenser/utils.py +++ b/openhands-sdk/openhands/sdk/context/condenser/utils.py @@ -1,8 +1,73 @@ from collections.abc import Sequence from openhands.sdk.event.base import LLMConvertibleEvent +from openhands.sdk.event.condenser import CondensationSummaryEvent +from openhands.sdk.event.llm_convertible import ( + ActionEvent, + AgentErrorEvent, + MessageEvent, + ObservationEvent, + UserRejectObservation, +) from openhands.sdk.event.llm_convertible.system import SystemPromptEvent -from openhands.sdk.llm import LLM +from openhands.sdk.llm import LLM, content_to_str +from openhands.sdk.utils import maybe_truncate + + +def render_event_for_summary( + event: LLMConvertibleEvent, max_length: int | None = None +) -> str: + """Render one forgotten event as the summarizing LLM should see it. + + Not ``str(event)``: that is a display preview capped at ``N_CHAR_PREVIEW`` + (500) characters, and it renders an action as its class name alone. Fed to + the summarizer it cut every previous summary to its first ~500 characters + and hid which command ran, which file was read and what a long user message + ended with -- so each condensation forgot the work the last one recorded, + and the agent repeated it. + + This shows what the agent itself was shown: a previous summary whole, an + action as its tool and arguments, a result, error or user message as its + text. ``max_length`` caps the rendered event, keeping head and tail; event + types not listed here keep their ``str()`` form. + """ + if isinstance(event, CondensationSummaryEvent): + text = f"PREVIOUS SUMMARY ({event.source})\n{event.summary}" + elif isinstance(event, ActionEvent): + lines = [f"ActionEvent ({event.source})", f" Tool: {event.tool_name}"] + if event.summary: + lines.append(f" Summary: {event.summary}") + thought = " ".join(t.text for t in event.thought).strip() + if thought: + lines.append(f" Thought: {thought}") + lines.append(f" Arguments: {event.tool_call.arguments}") + if event.action is None: + lines.append(" (not executed)") + text = "\n".join(lines) + elif isinstance(event, ObservationEvent): + result = "".join(content_to_str(event.observation.to_llm_content)) + text = ( + f"ObservationEvent ({event.source})\n" + f" Tool: {event.tool_name}\n Result: {result}" + ) + elif isinstance(event, UserRejectObservation): + text = ( + f"UserRejectObservation ({event.source})\n Tool: {event.tool_name}\n" + f" Rejected by {event.rejection_source}: {event.rejection_reason}" + ) + elif isinstance(event, AgentErrorEvent): + text = ( + f"AgentErrorEvent ({event.source})\n Tool: {event.tool_name}\n" + f" Error: {event.error}" + ) + elif isinstance(event, MessageEvent): + body = " ".join(content_to_str(event.llm_message.content)) or "[no text]" + text = f"MessageEvent ({event.source})\n {event.llm_message.role}: {body}" + if event.activated_skills: + text += f"\n [Skills: {', '.join(event.activated_skills)}]" + else: + text = str(event) + return maybe_truncate(text, truncate_after=max_length) def get_total_token_count( diff --git a/tests/sdk/agent/test_condensation_summary_masking.py b/tests/sdk/agent/test_condensation_summary_masking.py new file mode 100644 index 0000000000..c8c12ef67b --- /dev/null +++ b/tests/sdk/agent/test_condensation_summary_masking.py @@ -0,0 +1,95 @@ +"""A condensation summary is durable model output: registered secrets are masked. + +The summarizer is shown each forgotten action's arguments verbatim -- including a +token the model typed into a command, such as a git remote URL. Whatever it +copies into the summary is persisted and replayed to the agent on every later +step, so the agent masks it before emitting the condensation, exactly as it masks +its own messages (#4783). +""" + +import pytest + +from openhands.sdk.agent import Agent +from openhands.sdk.context.condenser.base import CondenserBase +from openhands.sdk.context.view import View +from openhands.sdk.conversation import Conversation, LocalConversation +from openhands.sdk.event.base import Event +from openhands.sdk.event.condenser import Condensation +from openhands.sdk.llm import LLM + + +SECRET_VALUE = "glpat-condenser-leak-0123456789" + + +class EchoingCondenser(CondenserBase): + """Condenses on every step with a summary that repeats a secret.""" + + summary: str | None + + def condense(self, view: View, agent_llm: LLM | None = None) -> View | Condensation: + return Condensation( + forgotten_event_ids=set(), + summary=self.summary, + summary_offset=0 if self.summary is not None else None, + llm_response_id="summary-response", + ) + + +def _conversation(summary: str | None) -> tuple[Agent, LocalConversation]: + agent = Agent( + llm=LLM(model="test-model", usage_id="test-llm"), + tools=[], + condenser=EchoingCondenser(summary=summary), + ) + conversation = Conversation(agent=agent) + assert isinstance(conversation, LocalConversation) + conversation._ensure_agent_ready() + conversation.update_secrets({"GITLAB_TOKEN": SECRET_VALUE}) + return agent, conversation + + +def _emitted_condensation(events: list[Event]) -> Condensation: + condensations = [e for e in events if isinstance(e, Condensation)] + assert len(condensations) == 1 + return condensations[0] + + +SUMMARY = ( + "CURRENT_STATE:\n- Re-cloned with " + f"`git clone https://oauth2:{SECRET_VALUE}@gitlab.com/acme/orders.git`" +) + + +def test_step_masks_secrets_in_the_condensation_summary() -> None: + agent, conversation = _conversation(SUMMARY) + events: list[Event] = [] + + agent.step(conversation, on_event=events.append) + + summary = _emitted_condensation(events).summary + assert summary is not None + assert SECRET_VALUE not in summary + assert "" in summary + assert "gitlab.com/acme/orders.git" in summary + + +@pytest.mark.asyncio +async def test_astep_masks_secrets_in_the_condensation_summary() -> None: + agent, conversation = _conversation(SUMMARY) + events: list[Event] = [] + + await agent.astep(conversation, on_event=events.append) + + summary = _emitted_condensation(events).summary + assert summary is not None + assert SECRET_VALUE not in summary + assert "" in summary + + +def test_step_leaves_a_condensation_without_summary_alone() -> None: + agent, conversation = _conversation(None) + events: list[Event] = [] + + agent.step(conversation, on_event=events.append) + + assert _emitted_condensation(events).summary is None diff --git a/tests/sdk/context/condenser/test_llm_summarizing_condenser.py b/tests/sdk/context/condenser/test_llm_summarizing_condenser.py index ec52962076..3059fc8fb9 100644 --- a/tests/sdk/context/condenser/test_llm_summarizing_condenser.py +++ b/tests/sdk/context/condenser/test_llm_summarizing_condenser.py @@ -1,3 +1,5 @@ +import json +from collections.abc import Sequence from typing import Any, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -21,14 +23,22 @@ from openhands.sdk.context.view import View from openhands.sdk.event.base import Event, LLMConvertibleEvent from openhands.sdk.event.condenser import Condensation, CondensationRequest -from openhands.sdk.event.llm_convertible import MessageEvent, SystemPromptEvent +from openhands.sdk.event.llm_convertible import ( + ActionEvent, + MessageEvent, + ObservationEvent, + SystemPromptEvent, +) from openhands.sdk.llm import ( LLM, + ImageContent, LLMResponse, Message, + MessageToolCall, MetricsSnapshot, TextContent, ) +from openhands.sdk.tool import Action, Observation def message_event(content: str) -> MessageEvent: @@ -1158,3 +1168,440 @@ def test_condensation_with_keep_first_zero_keeps_system_first(mock_llm: LLM) -> assert messages[0].role == "system" # The forgotten set must not include the leading system prompt event. assert events[0].id not in result.forgotten_event_ids + + +# --------------------------------------------------------------------------- +# What the summarizer is shown +# +# The summary is the agent's ONLY memory of every forgotten event, so what the +# summarizer is shown bounds what the agent can remember. These tests pin that +# input to the events themselves rather than to `str(event)`, which is a +# 500-character display preview: it cut a previous summary to its first ~500 +# characters (in practice, USER_CONTEXT alone) and rendered every action as its +# class name with no command, path or arguments. +# --------------------------------------------------------------------------- + + +class SummaryInputMockAction(Action): + command: str + + +class SummaryInputMockObservation(Observation): + result: str + + @property + def to_llm_content(self) -> Sequence[TextContent | ImageContent]: + return [TextContent(text=self.result)] + + +def _tool_pair( + call_id: str, + arguments: dict[str, Any], + result: str, + *, + thought: str = "", + summary: str | None = None, + tool_name: str = "terminal", +) -> list[Event]: + """An action and its observation, as the agent loop records them.""" + action = ActionEvent( + thought=[TextContent(text=thought)] if thought else [], + action=SummaryInputMockAction(command=str(arguments.get("command", ""))), + tool_name=tool_name, + tool_call_id=call_id, + tool_call=MessageToolCall( + id=call_id, + name=tool_name, + arguments=json.dumps(arguments), + origin="completion", + ), + llm_response_id=f"response-{call_id}", + summary=summary, + ) + observation = ObservationEvent( + observation=SummaryInputMockObservation(result=result), + action_id=action.id, + tool_name=tool_name, + tool_call_id=call_id, + ) + return [action, observation] + + +def _summarizer_prompt(mock_llm: LLM) -> str: + completion_mock = cast(MagicMock, mock_llm.generate) + completion_mock.assert_called_once() + messages = completion_mock.call_args[1]["messages"] + return "\n".join( + part.text + for message in messages + for part in message.content + if isinstance(part, TextContent) + ) + + +def _view_forgetting( + forgettable: Sequence[Event], previous_summary: str | None +) -> View: + """A view whose next condensation forgets every event in `forgettable`. + + keep_first=2 and max_size=10 keep two events at the head and two at the + tail; the previous summary, when given, sits at the head of the forgotten + range exactly where a real earlier condensation put it. + """ + head: list[Event] = [message_event("The task"), message_event("Head event")] + tail: list[Event] = [message_event(f"Tail {i}") for i in range(6)] + events: list[Event] = list(head) + if previous_summary is not None: + stale = message_event("Already summarised") + events.append(stale) + events.append( + Condensation( + forgotten_event_ids={stale.id}, + summary=previous_summary, + summary_offset=2, + llm_response_id="earlier-condensation", + ) + ) + return View.from_events([*events, *forgettable, *tail]) + + +def test_summarizer_sees_the_whole_previous_summary(mock_llm: LLM) -> None: + """Every condensation after the first re-summarises the previous summary. + + Cut to a preview, its COMPLETED / CODE_STATE / VERSION_CONTROL_STATUS + sections vanish and the next summary reports work already done as "not + checked yet" -- the agent then explores the same files again. + """ + previous = ( + "USER_CONTEXT:\n- Add workstations as first-class entities.\n" + + "COMPLETED:\n" + + "".join(f"- Reviewed models/Model{i}.js and its routes\n" for i in range(60)) + + "VERSION_CONTROL_STATUS:\n- Backend HEAD is d006196 on master\n" + ) + assert len(previous) > 2000 + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + + condenser.get_condensation( + _view_forgetting([message_event(f"Middle {i}") for i in range(4)], previous) + ) + + prompt = _summarizer_prompt(mock_llm) + assert "Backend HEAD is d006196 on master" in prompt + assert "Reviewed models/Model59.js" in prompt + + +def test_summarizer_sees_what_each_action_did(mock_llm: LLM) -> None: + """An action is its tool and arguments, not its class name. + + Most models emit tool calls with no thought text, so a preview of an action + was literally `Thought:` and `Action: SummaryInputMockAction` -- the + summarizer could not tell which command ran or which file was read. + """ + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + forgettable = _tool_pair( + "call-1", + {"command": "cat balapitiya_automobiles_backend/package.json"}, + '{"name": "balapitiya_automobiles_backend"}', + summary="Read backend package.json", + ) + [message_event("Middle")] + + condenser.get_condensation(_view_forgetting(forgettable, None)) + + prompt = _summarizer_prompt(mock_llm) + assert "cat balapitiya_automobiles_backend/package.json" in prompt + assert "Read backend package.json" in prompt + assert "terminal" in prompt + + +def test_summarizer_sees_a_long_user_message_whole(mock_llm: LLM) -> None: + """A user's answer at the end of a long message must reach the summary.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + answer = "Ticket id is AB-TEST-12" + long_message = "Some context about the change. " * 40 + answer + assert len(long_message) > 1000 + + condenser.get_condensation( + _view_forgetting( + [message_event(long_message)] + + [message_event(f"Middle {i}") for i in range(3)], + None, + ) + ) + + assert answer in _summarizer_prompt(mock_llm) + + +def test_summarizer_sees_the_tail_of_a_tool_result(mock_llm: LLM) -> None: + """The verdict of a command is usually its last line, past any preview.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + output = "> webpack compiling\n" * 150 + "npm ERR! Missing script: start" + assert len(output) > 2000 + + condenser.get_condensation( + _view_forgetting( + _tool_pair("call-1", {"command": "npm start"}, output) + + [message_event("Middle")], + None, + ) + ) + + assert "npm ERR! Missing script: start" in _summarizer_prompt(mock_llm) + + +def test_summarizer_input_stays_bounded_for_a_huge_tool_result(mock_llm: LLM) -> None: + """Whole events, but never an unbounded one: the head and tail survive.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + output = "FIRST LINE\n" + "x" * 200_000 + "\nLAST LINE" + + condenser.get_condensation( + _view_forgetting( + _tool_pair("call-1", {"command": "cat big.log"}, output) + + [message_event("Middle")], + None, + ) + ) + + prompt = _summarizer_prompt(mock_llm) + assert "FIRST LINE" in prompt + assert "LAST LINE" in prompt + assert len(prompt) < 20_000 + + +@pytest.mark.asyncio +async def test_async_summarizer_sees_the_whole_previous_summary( + mock_llm: LLM, +) -> None: + """The async path renders events the same way as the sync one.""" + previous = ( + "USER_CONTEXT:\n- Goal\n" + "x" * 3000 + "\nPENDING:\n- Write the model\n" + ) + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + forgettable = _tool_pair("call-1", {"command": "ls -la routes"}, "app.js") + forgettable.append(message_event("Middle")) + + await condenser.aget_condensation(_view_forgetting(forgettable, previous)) + + agenerate_mock = cast(AsyncMock, mock_llm.agenerate) + agenerate_mock.assert_awaited_once() + prompt = "\n".join( + part.text + for message in agenerate_mock.call_args[1]["messages"] + for part in message.content + if isinstance(part, TextContent) + ) + assert "- Write the model" in prompt + assert "ls -la routes" in prompt + + +def test_hard_context_reset_retry_shrinks_from_the_rendered_events( + mock_llm: LLM, +) -> None: + """A failed reset retries at 80% of the longest event, not of its preview. + + Scaling from the ~500-character preview cut every event to ~400 characters + on the first retry, however small the overflow was. + """ + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + fact = "the payment service listens on port 8443" + long_message = "Background. " * 150 + fact + "." + view = View.from_events( + [message_event("The task"), message_event(long_message), message_event("x")] + ) + ok = cast(MagicMock, mock_llm.generate).return_value + cast(MagicMock, mock_llm.generate).side_effect = [ + RuntimeError("context window exceeded"), + ok, + ] + + assert condenser.hard_context_reset(view) is not None + + retry_messages = cast(MagicMock, mock_llm.generate).call_args[1]["messages"] + retry_prompt = "\n".join( + part.text + for message in retry_messages + for part in message.content + if isinstance(part, TextContent) + ) + assert fact in retry_prompt + + +def test_summarizer_overflow_shrinks_events_instead_of_failing(mock_llm: LLM) -> None: + """Whole events may not fit a smaller summarizer window: shrink, then retry. + + Failing instead would fall through to a hard context reset, which forgets + every event -- far worse than a summary of trimmed ones. + """ + from openhands.sdk.llm.exceptions import LLMContextWindowExceedError + + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + ok = cast(MagicMock, mock_llm.generate).return_value + completion_mock = cast(MagicMock, mock_llm.generate) + completion_mock.side_effect = [LLMContextWindowExceedError("too long"), ok] + output = "START\n" + "y" * 8_000 + "\nEND" + + result = condenser.get_condensation( + _view_forgetting( + _tool_pair("call-1", {"command": "cat big"}, output) + + [message_event("Middle")], + None, + ) + ) + + assert isinstance(result, Condensation) + assert completion_mock.call_count == 2 + first, retry = ( + "\n".join( + part.text + for message in call[1]["messages"] + for part in message.content + if isinstance(part, TextContent) + ) + for call in completion_mock.call_args_list + ) + assert len(retry) < len(first) + assert "START" in retry and "END" in retry + + +def test_summarizer_failure_that_is_not_overflow_is_not_retried(mock_llm: LLM) -> None: + """An auth or provider error will not go away by trimming: raise at once.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + completion_mock = cast(MagicMock, mock_llm.generate) + completion_mock.side_effect = RuntimeError("401 unauthorized") + + with pytest.raises(NoCondensationAvailableException): + condenser.get_condensation( + _view_forgetting([message_event(f"Middle {i}") for i in range(4)], None) + ) + + assert completion_mock.call_count == 1 + + +def _overflow_above(limit: int, ok: LLMResponse): + """A summarizer whose window holds `limit` characters of prompt.""" + from openhands.sdk.llm.exceptions import LLMContextWindowExceedError + + def complete(*_args: Any, **kwargs: Any) -> LLMResponse: + size = sum( + len(part.text) + for message in kwargs["messages"] + for part in message.content + if isinstance(part, TextContent) + ) + if size > limit: + raise LLMContextWindowExceedError(f"{size} > {limit}") + return ok + + return complete + + +def _many_short_events_and_one_long() -> list[Event]: + """The shape a per-event cap alone cannot shrink: most events sit below it.""" + events: list[Event] = _tool_pair("call-big", {"command": "cat big"}, "z" * 9_000) + for i in range(40): + events += _tool_pair(f"call-{i}", {"command": f"cat f{i}"}, "w" * 1_200) + return events + + +def test_summarizer_overflow_shrinks_the_whole_prompt_not_one_event( + mock_llm: LLM, +) -> None: + """Each retry must cut the prompt, not only the longest event. + + Lowering the cap from the longest event leaves forty 1,200-character results + untouched, so every retry re-sends nearly the same prompt and the condenser + falls through to the hard reset the retry exists to avoid. + """ + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + view = _view_forgetting(_many_short_events_and_one_long(), None) + full = len("".join(condenser._render_forgotten_events(view.events))) + completion_mock = cast(MagicMock, mock_llm.generate) + completion_mock.side_effect = _overflow_above( + int(full * 0.6), completion_mock.return_value + ) + + result = condenser.get_condensation(view) + + assert isinstance(result, Condensation) + assert completion_mock.call_count <= condenser.hard_context_reset_max_retries + + +def test_summarizer_overflow_gives_up_when_nothing_is_left_to_cut( + mock_llm: LLM, +) -> None: + """A window that cannot hold even the trimmed events raises, never loops.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + completion_mock = cast(MagicMock, mock_llm.generate) + completion_mock.side_effect = _overflow_above(10, completion_mock.return_value) + + with pytest.raises(NoCondensationAvailableException): + condenser.get_condensation( + _view_forgetting(_many_short_events_and_one_long(), None) + ) + + sizes = [ + sum( + len(part.text) + for message in call[1]["messages"] + for part in message.content + if isinstance(part, TextContent) + ) + for call in completion_mock.call_args_list + ] + assert len(sizes) <= condenser.hard_context_reset_max_retries + assert sizes == sorted(sizes, reverse=True) + assert len(set(sizes)) == len(sizes), "a retry re-sent the same prompt" + + +@pytest.mark.asyncio +async def test_async_summarizer_overflow_shrinks_events_instead_of_failing( + mock_llm: LLM, +) -> None: + """Async variant of the overflow retry: shrink, then succeed.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + view = _view_forgetting(_many_short_events_and_one_long(), None) + full = len("".join(condenser._render_forgotten_events(view.events))) + sync_ok = cast(MagicMock, mock_llm.generate).return_value + agenerate_mock = AsyncMock(side_effect=_overflow_above(int(full * 0.6), sync_ok)) + cast(Any, mock_llm).agenerate = agenerate_mock + + result = await condenser.aget_condensation(view) + + assert isinstance(result, Condensation) + assert 1 < agenerate_mock.await_count <= condenser.hard_context_reset_max_retries + + +@pytest.mark.asyncio +async def test_async_summarizer_failure_that_is_not_overflow_is_not_retried( + mock_llm: LLM, +) -> None: + """Async variant: an auth or provider error raises at once.""" + condenser = LLMSummarizingCondenser(llm=mock_llm, max_size=10, keep_first=2) + agenerate_mock = AsyncMock(side_effect=RuntimeError("401 unauthorized")) + cast(Any, mock_llm).agenerate = agenerate_mock + + with pytest.raises(NoCondensationAvailableException): + await condenser.aget_condensation( + _view_forgetting([message_event(f"Middle {i}") for i in range(4)], None) + ) + + assert agenerate_mock.await_count == 1 + + +def test_summarizer_overflow_never_resends_untrimmed_events(mock_llm: LLM) -> None: + """A cap that rounds to 0 means "no truncation" downstream: raise instead. + + `maybe_truncate(text, 0)` returns the text whole, so retrying at a 0 cap would + re-send the full prompt that just overflowed. + """ + condenser = LLMSummarizingCondenser( + llm=mock_llm, max_size=10, keep_first=2, hard_context_reset_context_scaling=0.01 + ) + completion_mock = cast(MagicMock, mock_llm.generate) + completion_mock.side_effect = _overflow_above(10, completion_mock.return_value) + + with pytest.raises(NoCondensationAvailableException, match="nothing left to cut"): + condenser.get_condensation( + _view_forgetting([message_event(f"Middle {i}") for i in range(4)], None) + ) + + assert completion_mock.call_count == 1