fix(browsecomp): accept both vLLM /tokenize response shapes

responses_api_agents/browsecomp_agent/app.py

@@ -992,7 +992,25 @@ async def _count_prompt_tokens(self, body) -> int:
             if key in chat_completion_create_params:
                 tokenize_body_dict[key] = chat_completion_create_params[key]
         tokenize_response = await self._policy_model_openai_client.create_tokenize(**tokenize_body_dict)
-        return len(tokenize_response["tokens"])
+        return _prompt_tokens_from_tokenize_response(tokenize_response)
+
+
+def _prompt_tokens_from_tokenize_response(tokenize_response: dict) -> int:
+    """Prompt token count from a vLLM /tokenize response.
+
+    The response shape varies across vLLM versions: mainstream builds
+    (>=0.19.1) return {"count": N, "max_model_len": ...} and only include the
+    "tokens" list when token ids are requested, while older builds return just
+    {"tokens": [...]}. Prefer the explicit count and fall back to the token
+    list, so the agent works against both.
+    """
+    count = tokenize_response.get("count")
+    if count is not None:
+        return int(count)
+    tokens = tokenize_response.get("tokens")
+    if tokens is not None:
+        return len(tokens)
+    raise KeyError(f"/tokenize response has neither 'count' nor 'tokens'; got keys: {sorted(tokenize_response)}")
 
 
 if __name__ == "__main__":

responses_api_agents/browsecomp_agent/tests/test_app.py

@@ -478,3 +478,20 @@ def _http(read_bytes: bytes | None = None) -> MagicMock:
 
         assert agent.server_client.post.call_count == 4  # retry fired -> attempt 1 + verify
         assert result.reward == 1.0
+
+
+def test_prompt_tokens_from_tokenize_response_shapes():
+    """/tokenize responses differ across vLLM versions -- accept both."""
+    from responses_api_agents.browsecomp_agent.app import _prompt_tokens_from_tokenize_response
+
+    # Mainstream vLLM >=0.19.1: explicit count, no token list.
+    assert _prompt_tokens_from_tokenize_response({"count": 123, "max_model_len": 131072}) == 123
+    # Older builds: token id list only.
+    assert _prompt_tokens_from_tokenize_response({"tokens": [1, 2, 3]}) == 3
+    # Count wins when both are present.
+    assert _prompt_tokens_from_tokenize_response({"count": 5, "tokens": [1]}) == 5
+    # Neither -> loud failure.
+    import pytest
+
+    with pytest.raises(KeyError):
+        _prompt_tokens_from_tokenize_response({"max_model_len": 131072})