Add tests for json_utils.json_fix_llm (#2952)

* config.py: make load_dotenv(override=True)

* Update Python version and benchmark file in benchmark.yml

* Refactor main function and imports in cli.py

* Update import statement in ai_config.py

* Add set_temperature and set_memory_backend methods in config.py

* Remove unused import in prompt.py

* Add goal oriented tasks workflow

* Added agent_utils to create agent

* added pytest and vcrpy

* added write file cassette

* created goal oriented task write file with cassettes to not pay openai tokens

* solve conflicts

* add ability set azure because github workflow needs it off

* solve conflicts in cli.py

* black because linter fails

* solve conflict

* setup github action to v3

Signed-off-by: Merwane Hamadi <[email protected]>

* fix conflicts

Signed-off-by: Merwane Hamadi <[email protected]>

* Plugins: debug line always printed in plugin load

* add test for fix_json_using_multiple_techniques

* style

* style

* mocking try_ai_fix to avoid call_ai_function

* black style

* mock try_ai_fix to avoid calling the AI model

* removed mock, as we can add @requires_api_key("OPEN_API_KEY")

* style

* reverse merge conflict related files and changes

* bring back the mock for try_ai_fix

---------

Signed-off-by: Merwane Hamadi <[email protected]>
Co-authored-by: Merwane Hamadi <[email protected]>
Co-authored-by: Merwane Hamadi <[email protected]>
Co-authored-by: Richard Beales <[email protected]>
Co-authored-by: Nicholas Tindle <[email protected]>
Co-authored-by: BillSchumacher <[email protected]>

tests/unit/test_json_utils_llm.py

@@ -0,0 +1,117 @@
+# Generated by CodiumAI
+import pytest
+from loguru import logger
+
+from autogpt.json_utils.json_fix_llm import (
+    fix_and_parse_json,
+    fix_json_using_multiple_techniques,
+)
+from tests.utils import requires_api_key
+
+"""
+Code Analysis
+
+Objective:
+- The objective of the function is to fix a given JSON string to make it parseable and fully compliant with two techniques.
+
+Inputs:
+- The function takes in a string called 'assistant_reply', which is the JSON string to be fixed.
+
+Flow:
+- The function first calls the 'fix_and_parse_json' function to parse and print the Assistant response.
+- If the parsed JSON is an empty dictionary, the function calls the 'attempt_to_fix_json_by_finding_outermost_brackets' function to fix the JSON string.
+- If the parsed JSON is not an empty dictionary, the function returns the parsed JSON.
+- If the parsed JSON is an empty dictionary and cannot be fixed, the function logs an error and returns an empty dictionary.
+
+Outputs:
+- The main output of the function is a dictionary containing the fixed JSON string.
+
+Additional aspects:
+- The function uses two techniques to fix the JSON string: parsing and finding outermost brackets.
+- The function logs an error if the JSON string cannot be fixed and returns an empty dictionary.
+- The function uses the 'CFG' object to determine whether to speak the error message or not.
+"""
+
+
+class TestFixJsonUsingMultipleTechniques:
+    # Tests that the function successfully fixes and parses a JSON string that is already compliant with both techniques.
+    def test_fix_and_parse_json_happy_path(self):
+        # Happy path test case where the JSON string is already compliant with both techniques
+        json_string = '{"text": "Hello world", "confidence": 0.9}'
+        expected_output = {"text": "Hello world", "confidence": 0.9}
+        assert fix_json_using_multiple_techniques(json_string) == expected_output
+
+    # Tests that the function successfully fixes and parses a JSON string that contains only whitespace characters.
+    # @requires_api_key("OPEN_API_KEY")
+    def test_fix_and_parse_json_whitespace(self, mocker):
+        # Happy path test case where the JSON string contains only whitespace characters
+        json_string = "   \n\t   "
+
+        # mock try_ai_fix to avoid calling the AI model:
+        mocker.patch("autogpt.json_utils.json_fix_llm.try_ai_fix", return_value={})
+
+        expected_output = {}
+        assert fix_json_using_multiple_techniques(json_string) == expected_output
+
+    # Tests that the function successfully converts a string with arrays to an array
+    def test_fix_and_parse_json_array(self):
+        # Happy path test case where the JSON string contains an array of string
+        json_string = '[ "Add type hints", "Move docstrings", "Consider using" ]'
+        expected_output = ["Add type hints", "Move docstrings", "Consider using"]
+        assert fix_json_using_multiple_techniques(json_string) == expected_output
+
+    # Tests that the function returns an empty dictionary when the JSON string is not parseable and cannot be fixed using either technique.
+    # @requires_api_key("OPEN_API_KEY")
+    def test_fix_and_parse_json_can_not(self, mocker):
+        # Edge case test case where the JSON string is not parseable and cannot be fixed using either technique
+        json_string = "This is not a JSON string"
+
+        # mock try_ai_fix to avoid calling the AI model:
+        mocker.patch("autogpt.json_utils.json_fix_llm.try_ai_fix", return_value={})
+
+        expected_output = {}
+
+        # Use the actual function name in the test
+        result = fix_json_using_multiple_techniques(json_string)
+
+        assert result == expected_output
+
+    # Tests that the function returns an empty dictionary when the JSON string is empty.
+    # @requires_api_key("OPEN_API_KEY")
+    def test_fix_and_parse_json_empty_string(self, mocker):
+        # Arrange
+        json_string = ""
+
+        # Act
+        # mock try_ai_fix to avoid calling the AI model:
+        mocker.patch("autogpt.json_utils.json_fix_llm.try_ai_fix", return_value={})
+
+        result = fix_and_parse_json(json_string)
+
+        # Assert
+        assert result == {}
+
+    # Tests that the function successfully fixes and parses a JSON string that contains escape characters.
+    def test_fix_and_parse_json_escape_characters(self):
+        # Arrange
+        json_string = '{"text": "This is a \\"test\\" string."}'
+
+        # Act
+        result = fix_json_using_multiple_techniques(json_string)
+
+        # Assert
+        assert result == {"text": 'This is a "test" string.'}
+
+    # Tests that the function successfully fixes and parses a JSON string that contains nested objects or arrays.
+    def test_fix_and_parse_json_nested_objects(self):
+        # Arrange
+        json_string = '{"person": {"name": "John", "age": 30}, "hobbies": ["reading", "swimming"]}'
+
+        # Act
+        result = fix_json_using_multiple_techniques(json_string)
+
+        # Assert
+        assert result == {
+            "person": {"name": "John", "age": 30},
+            "hobbies": ["reading", "swimming"],
+        }