diff --git a/lib/crewai/src/crewai/agents/agent_builder/base_agent_executor_mixin.py b/lib/crewai/src/crewai/agents/agent_builder/base_agent_executor_mixin.py index 5864a4995e..30e078ced6 100644 --- a/lib/crewai/src/crewai/agents/agent_builder/base_agent_executor_mixin.py +++ b/lib/crewai/src/crewai/agents/agent_builder/base_agent_executor_mixin.py @@ -9,6 +9,7 @@ from crewai.utilities.converter import ConverterError from crewai.utilities.evaluators.task_evaluator import TaskEvaluator from crewai.utilities.printer import Printer +from crewai.utilities.types import LLMMessage if TYPE_CHECKING: diff --git a/lib/crewai/src/crewai/agents/crew_agent_executor.py b/lib/crewai/src/crewai/agents/crew_agent_executor.py index 8c1eb2c0e5..3565694bca 100644 --- a/lib/crewai/src/crewai/agents/crew_agent_executor.py +++ b/lib/crewai/src/crewai/agents/crew_agent_executor.py @@ -34,6 +34,7 @@ handle_unknown_error, has_reached_max_iterations, is_context_length_exceeded, + is_null_response_because_context_length_exceeded, process_llm_response, ) from crewai.utilities.constants import TRAINING_DATA_FILE @@ -41,6 +42,7 @@ from crewai.utilities.printer import Printer from crewai.utilities.tool_utils import execute_tool_and_check_finality from crewai.utilities.training_handler import CrewTrainingHandler +from crewai.utilities.types import LLMMessage if TYPE_CHECKING: @@ -274,7 +276,7 @@ def _invoke_loop(self) -> AgentFinish: if e.__class__.__module__.startswith("litellm"): # Do not retry on litellm errors raise e - if is_context_length_exceeded(e): + if is_context_length_exceeded(exception=e) or is_null_response_because_context_length_exceeded(exception=e, messages=self.messages, llm=self.llm ): handle_context_length( respect_context_window=self.respect_context_window, printer=self._printer, diff --git a/lib/crewai/src/crewai/lite_agent.py b/lib/crewai/src/crewai/lite_agent.py index 4314e900e8..3391da4473 100644 --- a/lib/crewai/src/crewai/lite_agent.py +++ b/lib/crewai/src/crewai/lite_agent.py @@ -55,6 +55,7 @@ handle_unknown_error, has_reached_max_iterations, is_context_length_exceeded, + is_null_response_because_context_length_exceeded, parse_tools, process_llm_response, render_text_description_and_args, @@ -569,7 +570,7 @@ def _invoke_loop(self) -> AgentFinish: if e.__class__.__module__.startswith("litellm"): # Do not retry on litellm errors raise e - if is_context_length_exceeded(e): + if is_context_length_exceeded(exception=e) or is_null_response_because_context_length_exceeded(exception=e, messages=self.messages, llm=self.llm ): handle_context_length( respect_context_window=self.respect_context_window, printer=self._printer, diff --git a/lib/crewai/src/crewai/utilities/agent_utils.py b/lib/crewai/src/crewai/utilities/agent_utils.py index 2c8122b99b..804f47c2e9 100644 --- a/lib/crewai/src/crewai/utilities/agent_utils.py +++ b/lib/crewai/src/crewai/utilities/agent_utils.py @@ -390,20 +390,50 @@ def handle_output_parser_exception( return formatted_answer -def is_context_length_exceeded(exception: Exception) -> bool: - """Check if the exception is due to context length exceeding. +def is_context_length_exceeded( + exception: Exception +) -> bool: + """ + Check if the exception is due to context length exceeding or + response is empty because context length exceeded. Args: exception: The exception to check + messages: Messages sent to the LLM + llm: The LLM instance Returns: - bool: True if the exception is due to context length exceeding + True if the exception is due to context length exceeding or + the response is empty because of context length. """ return LLMContextLengthExceededError(str(exception))._is_context_limit_error( str(exception) ) +def is_null_response_because_context_length_exceeded( + exception: Exception, + messages: list[LLMMessage], + llm: LLM | BaseLLM, +) -> bool: + """Check if the response is null/empty because context length excedded. + + Args: + exception: The exception to check + + Returns: + bool: True if the exception is due to context length exceeding + """ + messages_string = " ".join([message["content"] for message in messages]) + cut_size = llm.get_context_window_size() + + messages_groups = [ + {"content": messages_string[i : i + cut_size]} + for i in range(0, len(messages_string), cut_size) + ] + return ((len(messages_groups) > 1) and isinstance(exception, ValueError) and "None or empty" in str(exception)) + + def handle_context_length( respect_context_window: bool, printer: Printer, diff --git a/lib/crewai/src/crewai/utilities/evaluators/task_evaluator.py b/lib/crewai/src/crewai/utilities/evaluators/task_evaluator.py index 0d40b505a1..7af6b4cd5d 100644 --- a/lib/crewai/src/crewai/utilities/evaluators/task_evaluator.py +++ b/lib/crewai/src/crewai/utilities/evaluators/task_evaluator.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, Field +from crewai.agents.agent_builder.base_agent import BaseAgent from crewai.events.event_bus import crewai_event_bus from crewai.events.types.task_events import TaskEvaluationEvent from crewai.llm import LLM @@ -13,7 +14,6 @@ if TYPE_CHECKING: - from crewai.agent import Agent from crewai.task import Task @@ -56,7 +56,7 @@ class TaskEvaluator: original_agent: The agent to evaluate. """ - def __init__(self, original_agent: Agent) -> None: + def __init__(self, original_agent: BaseAgent) -> None: """Initializes the TaskEvaluator with the given LLM and agent. Args: diff --git a/lib/crewai/src/crewai/utilities/tool_utils.py b/lib/crewai/src/crewai/utilities/tool_utils.py index eb433c02c9..b0ab508c1c 100644 --- a/lib/crewai/src/crewai/utilities/tool_utils.py +++ b/lib/crewai/src/crewai/utilities/tool_utils.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from crewai.agents.agent_builder.base_agent import BaseAgent from crewai.agents.parser import AgentAction from crewai.agents.tools_handler import ToolsHandler from crewai.security.fingerprint import Fingerprint diff --git a/lib/crewai/tests/agents/test_agent.py b/lib/crewai/tests/agents/test_agent.py index 8f77e1ba41..f3d6dbe11c 100644 --- a/lib/crewai/tests/agents/test_agent.py +++ b/lib/crewai/tests/agents/test_agent.py @@ -1585,6 +1585,55 @@ def test_handle_context_length_exceeds_limit(): mock_summarize.assert_called_once() +def test_handle_context_length_exceeds_no_response(): + mock_agent_finish = MagicMock(spec=AgentFinish) + mock_agent_finish.output = "This is the final answer" + + llm = LLM(model="gpt-4o-mini",context_window_size = 2, api_key = "DUMMY") + llm.context_window_size = 2 # Manually overriding it to be 2 + + exception_to_be_raised = ValueError("Invalid response from LLM call - None or empty.") + + with patch("crewai.agents.crew_agent_executor.handle_max_iterations_exceeded", return_value = mock_agent_finish) as mock_handle_max_iterations_exceeded: + with patch("crewai.agents.crew_agent_executor.get_llm_response") as mock_get_llm_response: + with patch("crewai.agents.crew_agent_executor.is_null_response_because_context_length_exceeded", return_value = True) as mock_is_null_response_because_context_length_exceeded: + with patch("crewai.utilities.agent_utils.summarize_messages") as mock_summarize_messages: + mock_get_llm_response.side_effect = exception_to_be_raised + + agent = Agent( + role="test role", + goal="test goal", + backstory="test backstory", + respect_context_window=True, + llm=llm, + max_iter = 0 + ) + + task = Task(description="The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.",expected_output="The final answer") + result = agent.execute_task( + task=task, + ) + + mock_get_llm_response.assert_called_once() + mock_is_null_response_because_context_length_exceeded.assert_called_once_with( + exception = exception_to_be_raised, + messages=[{'role': 'system', 'content': 'You are test role. test backstory\nYour personal goal is: test goal\nTo give my best complete final answer to the task respond using the exact following format:\n\nThought: I now can give a great answer\nFinal Answer: Your final answer must be the great and the most complete as possible, it must be outcome described.\n\nI MUST use these formats, my job depends on it!'}, {'role': 'user', 'content': "\nCurrent Task: The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.\n\nThis is the expected criteria for your final answer: The final answer\nyou MUST return the actual complete content as the final answer, not a summary.\n\nBegin! This is VERY important to you, use the tools available and give your best Final Answer, your job depends on it!\n\nThought:"}], + llm = llm + ) + + mock_summarize_messages.assert_called_once() + + assert mock_is_null_response_because_context_length_exceeded( + exception = exception_to_be_raised, + messages=[{'role': 'system', 'content': 'You are test role. test backstory\nYour personal goal is: test goal\nTo give my best complete final answer to the task respond using the exact following format:\n\nThought: I now can give a great answer\nFinal Answer: Your final answer must be the great and the most complete as possible, it must be outcome described.\n\nI MUST use these formats, my job depends on it!'}, {'role': 'user', 'content': "\nCurrent Task: The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.\n\nThis is the expected criteria for your final answer: The final answer\nyou MUST return the actual complete content as the final answer, not a summary.\n\nBegin! This is VERY important to you, use the tools available and give your best Final Answer, your job depends on it!\n\nThought:"}], + llm = llm + ) is True + + mock_summarize_messages.assert_called_once() + mock_handle_max_iterations_exceeded.assert_called_once() + assert result == "This is the final answer" + + @pytest.mark.vcr(filter_headers=["authorization"]) def test_handle_context_length_exceeds_limit_cli_no(): agent = Agent( diff --git a/lib/crewai/tests/utilities/test_agent_utils.py b/lib/crewai/tests/utilities/test_agent_utils.py new file mode 100755 index 0000000000..7e99b4d2fc --- /dev/null +++ b/lib/crewai/tests/utilities/test_agent_utils.py @@ -0,0 +1,92 @@ +import pytest +from unittest.mock import MagicMock +from src.crewai.utilities.agent_utils import is_null_response_because_context_length_exceeded + +def test_is_null_response_because_context_length_exceeded_true(): + """ + Test that the function returns True when the exception is a ValueError + with 'None or empty' and there are messages. + """ + # Arrange + mock_llm = MagicMock() + mock_llm.get_context_window_size.return_value = 10 + exception = ValueError("Invalid response from LLM call - None or empty.") + messages = [{"content": "This is a test message."}] + + # Act + result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm) + + # Assert + assert result is True + + +def test_is_null_response_because_context_length_exceeded_false_wrong_exception(): + """ + Test that the function returns False when the exception is not a ValueError. + """ + # Arrange + mock_llm = MagicMock() + mock_llm.get_context_window_size.return_value = 10 + exception = TypeError("Some other error.") + messages = [{"content": "This is a test message."}] + + # Act + result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm) + + # Assert + assert result is False + + +def test_is_null_response_because_context_length_exceeded_false_wrong_message(): + """ + Test that the function returns False when the exception message does not + contain 'None or empty'. + """ + # Arrange + mock_llm = MagicMock() + mock_llm.get_context_window_size.return_value = 10 + exception = ValueError("Another value error.") + messages = [{"content": "This is a test message."}] + + # Act + result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm) + + # Assert + assert result is False + + +def test_is_null_response_because_context_length_exceeded_false_empty_messages(): + """ + Test that the function returns False when the messages list is empty. + """ + # Arrange + mock_llm = MagicMock() + mock_llm.get_context_window_size.return_value = 10 + exception = ValueError("Invalid response from LLM call - None or empty.") + messages = [] + + # Act + result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm) + + # Assert + assert result is False + + +def test_is_null_response_because_context_length_exceeded_false(): + """ + Test that the function returns True when the exception is a ValueError + with 'None or empty' and there are messages. + """ + # Arrange + mock_llm = MagicMock() + mock_llm.get_context_window_size.return_value = 50 + exception = ValueError("Invalid response from LLM call - None or empty.") + messages = [{"content": "This is a test message."}] + + # Act + result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm) + + # Assert + assert result is False + +