fix(api): keep Responses history system-free 🩷

gateway/platforms/api_server.py

@@ -4427,17 +4427,37 @@ def _build_response_conversation_history(
         agent_messages = result.get("messages") if isinstance(result, dict) else None
 
         if isinstance(agent_messages, list) and agent_messages:
+            # The agent's API transcript may include its private system prompt
+            # as a leading message.  ResponseStore is client conversation
+            # state: persisting system messages both leaks the prompt and makes
+            # prefix detection miss, duplicating the full transcript each turn.
+            system_prefix = 0
+            while (
+                system_prefix < len(agent_messages)
+                and isinstance(agent_messages[system_prefix], dict)
+                and agent_messages[system_prefix].get("role") == "system"
+            ):
+                system_prefix += 1
+            stored_messages = agent_messages[system_prefix:]
             turn_start = APIServerAdapter._response_messages_turn_start_index(
                 conversation_history,
                 user_message,
                 result,
             )
             if turn_start:
-                return list(agent_messages)
+                return stored_messages
+
+            # A legacy caller may return only this turn, but still include the
+            # private system prefix.  Its first non-system message is the
+            # current user turn, so preserve prior history without appending
+            # that user a second time.
+            current_user = {"role": "user", "content": user_message}
+            if stored_messages and stored_messages[0] == current_user:
+                return prior + stored_messages
 
             full_history = prior
             full_history.append(current_user)
-            full_history.extend(agent_messages)
+            full_history.extend(stored_messages)
             return full_history
 
         full_history = prior
@@ -4456,13 +4476,22 @@ def _response_messages_turn_start_index(
         if not isinstance(agent_messages, list) or not agent_messages:
             return 0
 
+        system_prefix = 0
+        while (
+            system_prefix < len(agent_messages)
+            and isinstance(agent_messages[system_prefix], dict)
+            and agent_messages[system_prefix].get("role") == "system"
+        ):
+            system_prefix += 1
+        transcript = agent_messages[system_prefix:]
+
         prior = list(conversation_history)
         current_user = {"role": "user", "content": user_message}
         expected_prefix = prior + [current_user]
-        if agent_messages[:len(expected_prefix)] == expected_prefix:
-            return len(expected_prefix)
-        if prior and agent_messages[:len(prior)] == prior:
-            return len(prior)
+        if transcript[:len(expected_prefix)] == expected_prefix:
+            return system_prefix + len(expected_prefix)
+        if prior and transcript[:len(prior)] == prior:
+            return system_prefix + len(prior)
         return 0
 
     @classmethod

tests/gateway/test_api_server.py

@@ -2013,7 +2013,10 @@ async def test_previous_response_id_stores_full_agent_transcript_once(self, adap
                 mock_run.return_value = (
                     {
                         "final_response": "2",
-                        "messages": list(first_history),
+                        "messages": [
+                            {"role": "system", "content": "private prompt"},
+                            *first_history,
+                        ],
                         "api_calls": 1,
                     },
                     {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
@@ -2036,7 +2039,10 @@ async def test_previous_response_id_stores_full_agent_transcript_once(self, adap
                 mock_run.return_value = (
                     {
                         "final_response": "3",
-                        "messages": list(second_history),
+                        "messages": [
+                            {"role": "system", "content": "private prompt"},
+                            *second_history,
+                        ],
                         "api_calls": 1,
                     },
                     {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
@@ -2058,6 +2064,69 @@ async def test_previous_response_id_stores_full_agent_transcript_once(self, adap
             assert stored_history.count(first_history[0]) == 1
             assert stored_history.count({"role": "user", "content": "Now add 1 more"}) == 1
 
+    def test_response_history_does_not_duplicate_when_transcript_has_system_message(self):
+        """A system-prefixed full transcript must remain linear and system-free."""
+        prior = [
+            {"role": "user", "content": "turn 1"},
+            {"role": "assistant", "content": "one"},
+        ]
+        result = {
+            "messages": [
+                {"role": "system", "content": "private prompt"},
+                *prior,
+                {"role": "user", "content": "turn 2"},
+                {"role": "assistant", "content": "two"},
+            ]
+        }
+
+        stored = APIServerAdapter._build_response_conversation_history(
+            prior, "turn 2", result, "two"
+        )
+
+        assert stored == [*prior, {"role": "user", "content": "turn 2"}, {"role": "assistant", "content": "two"}]
+        assert not any(message["role"] == "system" for message in stored)
+
+    def test_response_history_keeps_system_prefixed_turn_only_suffix(self):
+        """A system prefix alone must not misclassify a turn-only result."""
+        prior = [{"role": "user", "content": "turn 1"}]
+        result = {
+            "messages": [
+                {"role": "system", "content": "private prompt"},
+                {"role": "assistant", "content": "turn 2 answer"},
+            ]
+        }
+
+        stored = APIServerAdapter._build_response_conversation_history(
+            prior, "turn 2", result, "turn 2 answer"
+        )
+
+        assert stored == [
+            *prior,
+            {"role": "user", "content": "turn 2"},
+            {"role": "assistant", "content": "turn 2 answer"},
+        ]
+
+    def test_response_history_keeps_user_system_prefixed_turn_only_suffix_once(self):
+        """A system-prefixed suffix containing its user turn is not full history."""
+        prior = [{"role": "user", "content": "turn 1"}]
+        result = {
+            "messages": [
+                {"role": "system", "content": "private prompt"},
+                {"role": "user", "content": "turn 2"},
+                {"role": "assistant", "content": "turn 2 answer"},
+            ]
+        }
+
+        stored = APIServerAdapter._build_response_conversation_history(
+            prior, "turn 2", result, "turn 2 answer"
+        )
+
+        assert stored == [
+            *prior,
+            {"role": "user", "content": "turn 2"},
+            {"role": "assistant", "content": "turn 2 answer"},
+        ]
+
     @pytest.mark.asyncio
     async def test_previous_response_id_outputs_only_current_turn_items(self, adapter):
         """Response output must not replay previous tool artifacts."""
@@ -2090,7 +2159,9 @@ async def test_previous_response_id_outputs_only_current_turn_items(self, adapte
                 "session_id": "api-test-session",
             },
         )
-        full_agent_transcript = prior_history + [
+        full_agent_transcript = [
+            {"role": "system", "content": "private prompt"},
+            *prior_history,
             {"role": "user", "content": "Read new file"},
             {
                 "role": "assistant",