From 7a32e03bd537b3c2b98b9f55b2962a37a0c046c6 Mon Sep 17 00:00:00 2001 From: bingokon Date: Mon, 17 Apr 2023 00:48:53 +0100 Subject: [PATCH 1/4] refactoring the all json utilities --- autogpt/agent/agent.py | 4 +- autogpt/app.py | 2 +- autogpt/json_fixes/auto_fix.py | 53 ------ autogpt/json_fixes/bracket_termination.py | 36 ---- autogpt/json_fixes/escaping.py | 33 ---- autogpt/json_fixes/master_json_fix_method.py | 28 --- autogpt/json_fixes/missing_quotes.py | 27 --- autogpt/json_fixes/utilities.py | 20 --- .../{json_fixes => json_utils}/__init__.py | 0 .../parsing.py => json_utils/auto_fix.py} | 170 ++++++++++++++++-- .../llm_response_format_1.json | 0 .../utilities.py} | 23 ++- autogpt/logs.py | 5 +- tests/test_json_parser.py | 2 +- tests/unit/json_tests.py | 2 +- 15 files changed, 188 insertions(+), 217 deletions(-) delete mode 100644 autogpt/json_fixes/auto_fix.py delete mode 100644 autogpt/json_fixes/bracket_termination.py delete mode 100644 autogpt/json_fixes/escaping.py delete mode 100644 autogpt/json_fixes/master_json_fix_method.py delete mode 100644 autogpt/json_fixes/missing_quotes.py delete mode 100644 autogpt/json_fixes/utilities.py rename autogpt/{json_fixes => json_utils}/__init__.py (100%) rename autogpt/{json_fixes/parsing.py => json_utils/auto_fix.py} (51%) rename autogpt/{json_schemas => json_utils}/llm_response_format_1.json (100%) rename autogpt/{json_validation/validate_json.py => json_utils/utilities.py} (63%) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index dca614c7f239..6ec0a623811a 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -3,8 +3,8 @@ from autogpt.chat import chat_with_ai, create_chat_message from autogpt.config import Config -from autogpt.json_fixes.master_json_fix_method import fix_json_using_multiple_techniques -from autogpt.json_validation.validate_json import validate_json +from autogpt.json_utils.auto_fix import fix_json_using_multiple_techniques +from autogpt.json_utils.utilities import validate_json from autogpt.logs import logger, print_assistant_thoughts from autogpt.speech import say_text from autogpt.spinner import Spinner diff --git a/autogpt/app.py b/autogpt/app.py index 78b5bd2fdeb0..190f934b3925 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -18,7 +18,7 @@ search_files, write_to_file, ) -from autogpt.json_fixes.parsing import fix_and_parse_json +from autogpt.json_utils.auto_fix import fix_and_parse_json from autogpt.memory import get_memory from autogpt.processing.text import summarize_text from autogpt.speech import say_text diff --git a/autogpt/json_fixes/auto_fix.py b/autogpt/json_fixes/auto_fix.py deleted file mode 100644 index 9fcf909a49a8..000000000000 --- a/autogpt/json_fixes/auto_fix.py +++ /dev/null @@ -1,53 +0,0 @@ -"""This module contains the function to fix JSON strings using GPT-3.""" -import json - -from autogpt.llm_utils import call_ai_function -from autogpt.logs import logger -from autogpt.config import Config - -CFG = Config() - - -def fix_json(json_string: str, schema: str) -> str: - """Fix the given JSON string to make it parseable and fully compliant with - the provided schema. - - Args: - json_string (str): The JSON string to fix. - schema (str): The schema to use to fix the JSON. - Returns: - str: The fixed JSON string. - """ - # Try to fix the JSON using GPT: - function_string = "def fix_json(json_string: str, schema:str=None) -> str:" - args = [f"'''{json_string}'''", f"'''{schema}'''"] - description_string = ( - "This function takes a JSON string and ensures that it" - " is parseable and fully compliant with the provided schema. If an object" - " or field specified in the schema isn't contained within the correct JSON," - " it is omitted. The function also escapes any double quotes within JSON" - " string values to ensure that they are valid. If the JSON string contains" - " any None or NaN values, they are replaced with null before being parsed." - ) - - # If it doesn't already start with a "`", add one: - if not json_string.startswith("`"): - json_string = "```json\n" + json_string + "\n```" - result_string = call_ai_function( - function_string, args, description_string, model=CFG.fast_llm_model - ) - logger.debug("------------ JSON FIX ATTEMPT ---------------") - logger.debug(f"Original JSON: {json_string}") - logger.debug("-----------") - logger.debug(f"Fixed JSON: {result_string}") - logger.debug("----------- END OF FIX ATTEMPT ----------------") - - try: - json.loads(result_string) # just check the validity - return result_string - except json.JSONDecodeError: # noqa: E722 - # Get the call stack: - # import traceback - # call_stack = traceback.format_exc() - # print(f"Failed to fix JSON: '{json_string}' "+call_stack) - return "failed" diff --git a/autogpt/json_fixes/bracket_termination.py b/autogpt/json_fixes/bracket_termination.py deleted file mode 100644 index dd9a83764ebf..000000000000 --- a/autogpt/json_fixes/bracket_termination.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Fix JSON brackets.""" -from __future__ import annotations - -import contextlib -import json -from typing import Optional -from autogpt.config import Config - -CFG = Config() - - -def balance_braces(json_string: str) -> Optional[str]: - """ - Balance the braces in a JSON string. - - Args: - json_string (str): The JSON string. - - Returns: - str: The JSON string with braces balanced. - """ - - open_braces_count = json_string.count("{") - close_braces_count = json_string.count("}") - - while open_braces_count > close_braces_count: - json_string += "}" - close_braces_count += 1 - - while close_braces_count > open_braces_count: - json_string = json_string.rstrip("}") - close_braces_count -= 1 - - with contextlib.suppress(json.JSONDecodeError): - json.loads(json_string) - return json_string diff --git a/autogpt/json_fixes/escaping.py b/autogpt/json_fixes/escaping.py deleted file mode 100644 index 68eb1714517c..000000000000 --- a/autogpt/json_fixes/escaping.py +++ /dev/null @@ -1,33 +0,0 @@ -""" Fix invalid escape sequences in JSON strings. """ -import json - -from autogpt.config import Config -from autogpt.json_fixes.utilities import extract_char_position - -CFG = Config() - - -def fix_invalid_escape(json_to_load: str, error_message: str) -> str: - """Fix invalid escape sequences in JSON strings. - - Args: - json_to_load (str): The JSON string. - error_message (str): The error message from the JSONDecodeError - exception. - - Returns: - str: The JSON string with invalid escape sequences fixed. - """ - while error_message.startswith("Invalid \\escape"): - bad_escape_location = extract_char_position(error_message) - json_to_load = ( - json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1 :] - ) - try: - json.loads(json_to_load) - return json_to_load - except json.JSONDecodeError as e: - if CFG.debug_mode: - print("json loads error - fix invalid escape", e) - error_message = str(e) - return json_to_load diff --git a/autogpt/json_fixes/master_json_fix_method.py b/autogpt/json_fixes/master_json_fix_method.py deleted file mode 100644 index 7a2cf3cc81c3..000000000000 --- a/autogpt/json_fixes/master_json_fix_method.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Any, Dict - -from autogpt.config import Config -from autogpt.logs import logger -from autogpt.speech import say_text -CFG = Config() - - -def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: - from autogpt.json_fixes.parsing import attempt_to_fix_json_by_finding_outermost_brackets - - from autogpt.json_fixes.parsing import fix_and_parse_json - - # Parse and print Assistant response - assistant_reply_json = fix_and_parse_json(assistant_reply) - if assistant_reply_json == {}: - assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets( - assistant_reply - ) - - if assistant_reply_json != {}: - return assistant_reply_json - - logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) - if CFG.speak_mode: - say_text("I have received an invalid JSON response from the OpenAI API.") - - return {} diff --git a/autogpt/json_fixes/missing_quotes.py b/autogpt/json_fixes/missing_quotes.py deleted file mode 100644 index 552a15173d09..000000000000 --- a/autogpt/json_fixes/missing_quotes.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Fix quotes in a JSON string.""" -import json -import re - - -def add_quotes_to_property_names(json_string: str) -> str: - """ - Add quotes to property names in a JSON string. - - Args: - json_string (str): The JSON string. - - Returns: - str: The JSON string with quotes added to property names. - """ - - def replace_func(match: re.Match) -> str: - return f'"{match[1]}":' - - property_name_pattern = re.compile(r"(\w+):") - corrected_json_string = property_name_pattern.sub(replace_func, json_string) - - try: - json.loads(corrected_json_string) - return corrected_json_string - except json.JSONDecodeError as e: - raise e diff --git a/autogpt/json_fixes/utilities.py b/autogpt/json_fixes/utilities.py deleted file mode 100644 index 0852b18a7a99..000000000000 --- a/autogpt/json_fixes/utilities.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Utilities for the json_fixes package.""" -import re - - -def extract_char_position(error_message: str) -> int: - """Extract the character position from the JSONDecodeError message. - - Args: - error_message (str): The error message from the JSONDecodeError - exception. - - Returns: - int: The character position. - """ - - char_pattern = re.compile(r"\(char (\d+)\)") - if match := char_pattern.search(error_message): - return int(match[1]) - else: - raise ValueError("Character position not found in the error message.") diff --git a/autogpt/json_fixes/__init__.py b/autogpt/json_utils/__init__.py similarity index 100% rename from autogpt/json_fixes/__init__.py rename to autogpt/json_utils/__init__.py diff --git a/autogpt/json_fixes/parsing.py b/autogpt/json_utils/auto_fix.py similarity index 51% rename from autogpt/json_fixes/parsing.py rename to autogpt/json_utils/auto_fix.py index 1e391eed7c02..883eba78bcb4 100644 --- a/autogpt/json_fixes/parsing.py +++ b/autogpt/json_utils/auto_fix.py @@ -1,20 +1,19 @@ -"""Fix and parse JSON strings.""" +"""This module contains the function to fix JSON strings""" from __future__ import annotations import contextlib import json -from typing import Any, Dict, Union +import re +from typing import Optional, Dict, Any + from colorama import Fore from regex import regex -from autogpt.config import Config -from autogpt.json_fixes.auto_fix import fix_json -from autogpt.json_fixes.bracket_termination import balance_braces -from autogpt.json_fixes.escaping import fix_invalid_escape -from autogpt.json_fixes.missing_quotes import add_quotes_to_property_names + +from autogpt.json_utils.utilities import extract_char_position +from autogpt.llm_utils import call_ai_function from autogpt.logs import logger from autogpt.speech import say_text - -CFG = Config() +from autogpt.config import Config JSON_SCHEMA = """ { @@ -35,6 +34,157 @@ } """ +CFG = Config() + + +def auto_fix_json(json_string: str, schema: str) -> str: + """Fix the given JSON string to make it parseable and fully compliant with + the provided schema using GPT-3. + + Args: + json_string (str): The JSON string to fix. + schema (str): The schema to use to fix the JSON. + Returns: + str: The fixed JSON string. + """ + # Try to fix the JSON using GPT: + function_string = "def fix_json(json_string: str, schema:str=None) -> str:" + args = [f"'''{json_string}'''", f"'''{schema}'''"] + description_string = ( + "This function takes a JSON string and ensures that it" + " is parseable and fully compliant with the provided schema. If an object" + " or field specified in the schema isn't contained within the correct JSON," + " it is omitted. The function also escapes any double quotes within JSON" + " string values to ensure that they are valid. If the JSON string contains" + " any None or NaN values, they are replaced with null before being parsed." + ) + + # If it doesn't already start with a "`", add one: + if not json_string.startswith("`"): + json_string = "```json\n" + json_string + "\n```" + result_string = call_ai_function( + function_string, args, description_string, model=CFG.fast_llm_model + ) + logger.debug("------------ JSON FIX ATTEMPT ---------------") + logger.debug(f"Original JSON: {json_string}") + logger.debug("-----------") + logger.debug(f"Fixed JSON: {result_string}") + logger.debug("----------- END OF FIX ATTEMPT ----------------") + + try: + json.loads(result_string) # just check the validity + return result_string + except json.JSONDecodeError: # noqa: E722 + # Get the call stack: + # import traceback + # call_stack = traceback.format_exc() + # print(f"Failed to fix JSON: '{json_string}' "+call_stack) + return "failed" + + +def fix_invalid_escape(json_to_load: str, error_message: str) -> str: + """Fix invalid escape sequences in JSON strings. + + Args: + json_to_load (str): The JSON string. + error_message (str): The error message from the JSONDecodeError + exception. + + Returns: + str: The JSON string with invalid escape sequences fixed. + """ + while error_message.startswith("Invalid \\escape"): + bad_escape_location = extract_char_position(error_message) + json_to_load = ( + json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1:] + ) + try: + json.loads(json_to_load) + return json_to_load + except json.JSONDecodeError as e: + if CFG.debug_mode: + print("json loads error - fix invalid escape", e) + error_message = str(e) + return json_to_load + + +def balance_braces(json_string: str) -> Optional[str]: + """ + Balance the braces in a JSON string. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with braces balanced. + """ + + open_braces_count = json_string.count("{") + close_braces_count = json_string.count("}") + + while open_braces_count > close_braces_count: + json_string += "}" + close_braces_count += 1 + + while close_braces_count > open_braces_count: + json_string = json_string.rstrip("}") + close_braces_count -= 1 + + with contextlib.suppress(json.JSONDecodeError): + json.loads(json_string) + return json_string + + +def add_quotes_to_property_names(json_string: str) -> str: + """ + Add quotes to property names in a JSON string. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with quotes added to property names. + """ + + def replace_func(match: re.Match) -> str: + return f'"{match[1]}":' + + property_name_pattern = re.compile(r"(\w+):") + corrected_json_string = property_name_pattern.sub(replace_func, json_string) + + try: + json.loads(corrected_json_string) + return corrected_json_string + except json.JSONDecodeError as e: + raise e + + +def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: + """Fix the given JSON string to make it parseable and fully compliant with two techniques. + + Args: + json_string (str): The JSON string to fix. + + Returns: + str: The fixed JSON string. + """ + + # Parse and print Assistant response + assistant_reply_json = fix_and_parse_json(assistant_reply) + if assistant_reply_json == {}: + assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets( + assistant_reply + ) + + if assistant_reply_json != {}: + return assistant_reply_json + + logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) + if CFG.speak_mode: + say_text("I have received an invalid JSON response from the OpenAI API.") + + return {} + def correct_json(json_to_load: str) -> str: """ @@ -134,7 +284,7 @@ def try_ai_fix( " slightly." ) # Now try to fix this up using the ai_functions - ai_fixed_json = fix_json(json_to_load, JSON_SCHEMA) + ai_fixed_json = auto_fix_json(json_to_load, JSON_SCHEMA) if ai_fixed_json != "failed": return json.loads(ai_fixed_json) diff --git a/autogpt/json_schemas/llm_response_format_1.json b/autogpt/json_utils/llm_response_format_1.json similarity index 100% rename from autogpt/json_schemas/llm_response_format_1.json rename to autogpt/json_utils/llm_response_format_1.json diff --git a/autogpt/json_validation/validate_json.py b/autogpt/json_utils/utilities.py similarity index 63% rename from autogpt/json_validation/validate_json.py rename to autogpt/json_utils/utilities.py index 440c3b0b9199..af8a28c96e79 100644 --- a/autogpt/json_validation/validate_json.py +++ b/autogpt/json_utils/utilities.py @@ -1,10 +1,31 @@ +"""Utilities for the json_fixes package.""" import json +import re + from jsonschema import Draft7Validator -from autogpt.config import Config + from autogpt.logs import logger +from autogpt.config import Config CFG = Config() +def extract_char_position(error_message: str) -> int: + """Extract the character position from the JSONDecodeError message. + + Args: + error_message (str): The error message from the JSONDecodeError + exception. + + Returns: + int: The character position. + """ + + char_pattern = re.compile(r"\(char (\d+)\)") + if match := char_pattern.search(error_message): + return int(match[1]) + else: + raise ValueError("Character position not found in the error message.") + def validate_json(json_object: object, schema_name: object) -> object: """ diff --git a/autogpt/logs.py b/autogpt/logs.py index c1e436db97fc..58375f14aef8 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -204,10 +204,7 @@ def remove_color_codes(s: str) -> str: def print_assistant_thoughts(ai_name, assistant_reply): """Prints the assistant's thoughts to the console""" - from autogpt.json_fixes.bracket_termination import ( - attempt_to_fix_json_by_finding_outermost_brackets, - ) - from autogpt.json_fixes.parsing import fix_and_parse_json + from autogpt.json_utils.auto_fix import fix_and_parse_json, attempt_to_fix_json_by_finding_outermost_brackets try: try: diff --git a/tests/test_json_parser.py b/tests/test_json_parser.py index 2862034b3fdd..f8fa59555b3b 100644 --- a/tests/test_json_parser.py +++ b/tests/test_json_parser.py @@ -1,7 +1,7 @@ import unittest import tests.context -from autogpt.json_fixes.parsing import fix_and_parse_json +from autogpt.json_utils.auto_fix import fix_and_parse_json class TestParseJson(unittest.TestCase): diff --git a/tests/unit/json_tests.py b/tests/unit/json_tests.py index 561b8a38a857..f65a6f6aff0b 100644 --- a/tests/unit/json_tests.py +++ b/tests/unit/json_tests.py @@ -1,6 +1,6 @@ import unittest -from autogpt.json_parser import fix_and_parse_json +from autogpt.json_utils.auto_fix import fix_and_parse_json class TestParseJson(unittest.TestCase): From 0d2e1963682e0e6a65934f475442637277266d03 Mon Sep 17 00:00:00 2001 From: BingokoN Date: Mon, 17 Apr 2023 12:14:43 +0100 Subject: [PATCH 2/4] refactoring/splitting the json fix functions into general module and llm module which need AI's assistance. --- autogpt/agent/agent.py | 2 +- autogpt/app.py | 1 - autogpt/json_utils/json_fix_general.py | 124 ++++++++++++++++++ .../{auto_fix.py => json_fix_llm.py} | 119 +---------------- autogpt/json_utils/utilities.py | 4 +- autogpt/logs.py | 2 +- tests/test_json_parser.py | 2 +- tests/unit/json_tests.py | 2 +- 8 files changed, 135 insertions(+), 121 deletions(-) create mode 100644 autogpt/json_utils/json_fix_general.py rename autogpt/json_utils/{auto_fix.py => json_fix_llm.py} (67%) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 6ec0a623811a..f87cd4830b71 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -3,7 +3,7 @@ from autogpt.chat import chat_with_ai, create_chat_message from autogpt.config import Config -from autogpt.json_utils.auto_fix import fix_json_using_multiple_techniques +from autogpt.json_utils.json_fix_llm import fix_json_using_multiple_techniques from autogpt.json_utils.utilities import validate_json from autogpt.logs import logger, print_assistant_thoughts from autogpt.speech import say_text diff --git a/autogpt/app.py b/autogpt/app.py index 190f934b3925..48db03665a88 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -18,7 +18,6 @@ search_files, write_to_file, ) -from autogpt.json_utils.auto_fix import fix_and_parse_json from autogpt.memory import get_memory from autogpt.processing.text import summarize_text from autogpt.speech import say_text diff --git a/autogpt/json_utils/json_fix_general.py b/autogpt/json_utils/json_fix_general.py new file mode 100644 index 000000000000..cd6a68841181 --- /dev/null +++ b/autogpt/json_utils/json_fix_general.py @@ -0,0 +1,124 @@ +"""This module contains functions to fix JSON strings using general programmatic approaches, suitable for addressing +common JSON formatting issues.""" +from __future__ import annotations + +import contextlib +import json +import re +from typing import Optional + +from autogpt.config import Config +from autogpt.json_utils.utilities import extract_char_position + +CFG = Config() + + +def fix_invalid_escape(json_to_load: str, error_message: str) -> str: + """Fix invalid escape sequences in JSON strings. + + Args: + json_to_load (str): The JSON string. + error_message (str): The error message from the JSONDecodeError + exception. + + Returns: + str: The JSON string with invalid escape sequences fixed. + """ + while error_message.startswith("Invalid \\escape"): + bad_escape_location = extract_char_position(error_message) + json_to_load = ( + json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1:] + ) + try: + json.loads(json_to_load) + return json_to_load + except json.JSONDecodeError as e: + if CFG.debug_mode: + print("json loads error - fix invalid escape", e) + error_message = str(e) + return json_to_load + + +def balance_braces(json_string: str) -> Optional[str]: + """ + Balance the braces in a JSON string. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with braces balanced. + """ + + open_braces_count = json_string.count("{") + close_braces_count = json_string.count("}") + + while open_braces_count > close_braces_count: + json_string += "}" + close_braces_count += 1 + + while close_braces_count > open_braces_count: + json_string = json_string.rstrip("}") + close_braces_count -= 1 + + with contextlib.suppress(json.JSONDecodeError): + json.loads(json_string) + return json_string + + +def add_quotes_to_property_names(json_string: str) -> str: + """ + Add quotes to property names in a JSON string. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with quotes added to property names. + """ + + def replace_func(match: re.Match) -> str: + return f'"{match[1]}":' + + property_name_pattern = re.compile(r"(\w+):") + corrected_json_string = property_name_pattern.sub(replace_func, json_string) + + try: + json.loads(corrected_json_string) + return corrected_json_string + except json.JSONDecodeError as e: + raise e + + +def correct_json(json_to_load: str) -> str: + """ + Correct common JSON errors. + Args: + json_to_load (str): The JSON string. + """ + + try: + if CFG.debug_mode: + print("json", json_to_load) + json.loads(json_to_load) + return json_to_load + except json.JSONDecodeError as e: + if CFG.debug_mode: + print("json loads error", e) + error_message = str(e) + if error_message.startswith("Invalid \\escape"): + json_to_load = fix_invalid_escape(json_to_load, error_message) + if error_message.startswith( + "Expecting property name enclosed in double quotes" + ): + json_to_load = add_quotes_to_property_names(json_to_load) + try: + json.loads(json_to_load) + return json_to_load + except json.JSONDecodeError as e: + if CFG.debug_mode: + print("json loads error - add quotes", e) + error_message = str(e) + if balanced_str := balance_braces(json_to_load): + return balanced_str + return json_to_load diff --git a/autogpt/json_utils/auto_fix.py b/autogpt/json_utils/json_fix_llm.py similarity index 67% rename from autogpt/json_utils/auto_fix.py rename to autogpt/json_utils/json_fix_llm.py index 883eba78bcb4..44e78d05257f 100644 --- a/autogpt/json_utils/auto_fix.py +++ b/autogpt/json_utils/json_fix_llm.py @@ -1,15 +1,15 @@ -"""This module contains the function to fix JSON strings""" +"""This module contains functions to fix JSON strings generated by LLM models, such as ChatGPT, using the assistance +of the ChatGPT API or LLM models.""" from __future__ import annotations import contextlib import json -import re -from typing import Optional, Dict, Any +from typing import Dict, Any from colorama import Fore from regex import regex -from autogpt.json_utils.utilities import extract_char_position +from autogpt.json_utils.json_fix_general import correct_json from autogpt.llm_utils import call_ai_function from autogpt.logs import logger from autogpt.speech import say_text @@ -82,83 +82,6 @@ def auto_fix_json(json_string: str, schema: str) -> str: return "failed" -def fix_invalid_escape(json_to_load: str, error_message: str) -> str: - """Fix invalid escape sequences in JSON strings. - - Args: - json_to_load (str): The JSON string. - error_message (str): The error message from the JSONDecodeError - exception. - - Returns: - str: The JSON string with invalid escape sequences fixed. - """ - while error_message.startswith("Invalid \\escape"): - bad_escape_location = extract_char_position(error_message) - json_to_load = ( - json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1:] - ) - try: - json.loads(json_to_load) - return json_to_load - except json.JSONDecodeError as e: - if CFG.debug_mode: - print("json loads error - fix invalid escape", e) - error_message = str(e) - return json_to_load - - -def balance_braces(json_string: str) -> Optional[str]: - """ - Balance the braces in a JSON string. - - Args: - json_string (str): The JSON string. - - Returns: - str: The JSON string with braces balanced. - """ - - open_braces_count = json_string.count("{") - close_braces_count = json_string.count("}") - - while open_braces_count > close_braces_count: - json_string += "}" - close_braces_count += 1 - - while close_braces_count > open_braces_count: - json_string = json_string.rstrip("}") - close_braces_count -= 1 - - with contextlib.suppress(json.JSONDecodeError): - json.loads(json_string) - return json_string - - -def add_quotes_to_property_names(json_string: str) -> str: - """ - Add quotes to property names in a JSON string. - - Args: - json_string (str): The JSON string. - - Returns: - str: The JSON string with quotes added to property names. - """ - - def replace_func(match: re.Match) -> str: - return f'"{match[1]}":' - - property_name_pattern = re.compile(r"(\w+):") - corrected_json_string = property_name_pattern.sub(replace_func, json_string) - - try: - json.loads(corrected_json_string) - return corrected_json_string - except json.JSONDecodeError as e: - raise e - - def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: """Fix the given JSON string to make it parseable and fully compliant with two techniques. @@ -186,40 +109,6 @@ def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: return {} -def correct_json(json_to_load: str) -> str: - """ - Correct common JSON errors. - Args: - json_to_load (str): The JSON string. - """ - - try: - if CFG.debug_mode: - print("json", json_to_load) - json.loads(json_to_load) - return json_to_load - except json.JSONDecodeError as e: - if CFG.debug_mode: - print("json loads error", e) - error_message = str(e) - if error_message.startswith("Invalid \\escape"): - json_to_load = fix_invalid_escape(json_to_load, error_message) - if error_message.startswith( - "Expecting property name enclosed in double quotes" - ): - json_to_load = add_quotes_to_property_names(json_to_load) - try: - json.loads(json_to_load) - return json_to_load - except json.JSONDecodeError as e: - if CFG.debug_mode: - print("json loads error - add quotes", e) - error_message = str(e) - if balanced_str := balance_braces(json_to_load): - return balanced_str - return json_to_load - - def fix_and_parse_json( json_to_load: str, try_to_fix_with_gpt: bool = True ) -> Dict[Any, Any]: diff --git a/autogpt/json_utils/utilities.py b/autogpt/json_utils/utilities.py index af8a28c96e79..e963abb37536 100644 --- a/autogpt/json_utils/utilities.py +++ b/autogpt/json_utils/utilities.py @@ -9,6 +9,7 @@ CFG = Config() + def extract_char_position(error_message: str) -> int: """Extract the character position from the JSONDecodeError message. @@ -40,7 +41,8 @@ def validate_json(json_object: object, schema_name: object) -> object: if errors := sorted(validator.iter_errors(json_object), key=lambda e: e.path): logger.error("The JSON object is invalid.") if CFG.debug_mode: - logger.error(json.dumps(json_object, indent=4)) # Replace 'json_object' with the variable containing the JSON data + logger.error( + json.dumps(json_object, indent=4)) # Replace 'json_object' with the variable containing the JSON data logger.error("The following issues were found:") for error in errors: diff --git a/autogpt/logs.py b/autogpt/logs.py index 58375f14aef8..a4dc3bab597d 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -204,7 +204,7 @@ def remove_color_codes(s: str) -> str: def print_assistant_thoughts(ai_name, assistant_reply): """Prints the assistant's thoughts to the console""" - from autogpt.json_utils.auto_fix import fix_and_parse_json, attempt_to_fix_json_by_finding_outermost_brackets + from autogpt.json_utils.json_fix_llm import fix_and_parse_json, attempt_to_fix_json_by_finding_outermost_brackets try: try: diff --git a/tests/test_json_parser.py b/tests/test_json_parser.py index f8fa59555b3b..41c90a6f66c0 100644 --- a/tests/test_json_parser.py +++ b/tests/test_json_parser.py @@ -1,7 +1,7 @@ import unittest import tests.context -from autogpt.json_utils.auto_fix import fix_and_parse_json +from autogpt.json_utils.json_fix_llm import fix_and_parse_json class TestParseJson(unittest.TestCase): diff --git a/tests/unit/json_tests.py b/tests/unit/json_tests.py index f65a6f6aff0b..25c383377708 100644 --- a/tests/unit/json_tests.py +++ b/tests/unit/json_tests.py @@ -1,6 +1,6 @@ import unittest -from autogpt.json_utils.auto_fix import fix_and_parse_json +from autogpt.json_utils.json_fix_llm import fix_and_parse_json class TestParseJson(unittest.TestCase): From 6787c2eeed703ff631323e3a6ea4d7541da14d4f Mon Sep 17 00:00:00 2001 From: bingokon Date: Tue, 18 Apr 2023 00:17:42 +0100 Subject: [PATCH 3/4] fix json_schemas not found error --- autogpt/json_utils/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/json_utils/utilities.py b/autogpt/json_utils/utilities.py index e5b8eb4a2c82..8499ddc853d7 100644 --- a/autogpt/json_utils/utilities.py +++ b/autogpt/json_utils/utilities.py @@ -34,7 +34,7 @@ def validate_json(json_object: object, schema_name: object) -> object: :param schema_name: :type json_object: object """ - with open(f"autogpt/json_schemas/{schema_name}.json", "r") as f: + with open(f"autogpt/json_utils/{schema_name}.json", "r") as f: schema = json.load(f) validator = Draft7Validator(schema) From 6e94409594b04e91a2e883daad4bdbea0c53e360 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Tue, 18 Apr 2023 19:40:14 -0500 Subject: [PATCH 4/4] linting --- autogpt/json_utils/json_fix_general.py | 4 ++-- autogpt/json_utils/json_fix_llm.py | 9 ++++++--- autogpt/json_utils/utilities.py | 2 +- autogpt/logs.py | 5 ++++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/autogpt/json_utils/json_fix_general.py b/autogpt/json_utils/json_fix_general.py index cd6a68841181..7010fa3b9c19 100644 --- a/autogpt/json_utils/json_fix_general.py +++ b/autogpt/json_utils/json_fix_general.py @@ -27,7 +27,7 @@ def fix_invalid_escape(json_to_load: str, error_message: str) -> str: while error_message.startswith("Invalid \\escape"): bad_escape_location = extract_char_position(error_message) json_to_load = ( - json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1:] + json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1 :] ) try: json.loads(json_to_load) @@ -109,7 +109,7 @@ def correct_json(json_to_load: str) -> str: if error_message.startswith("Invalid \\escape"): json_to_load = fix_invalid_escape(json_to_load, error_message) if error_message.startswith( - "Expecting property name enclosed in double quotes" + "Expecting property name enclosed in double quotes" ): json_to_load = add_quotes_to_property_names(json_to_load) try: diff --git a/autogpt/json_utils/json_fix_llm.py b/autogpt/json_utils/json_fix_llm.py index 44e78d05257f..869aed125cfb 100644 --- a/autogpt/json_utils/json_fix_llm.py +++ b/autogpt/json_utils/json_fix_llm.py @@ -4,16 +4,16 @@ import contextlib import json -from typing import Dict, Any +from typing import Any, Dict from colorama import Fore from regex import regex +from autogpt.config import Config from autogpt.json_utils.json_fix_general import correct_json from autogpt.llm_utils import call_ai_function from autogpt.logs import logger from autogpt.speech import say_text -from autogpt.config import Config JSON_SCHEMA = """ { @@ -102,7 +102,10 @@ def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: if assistant_reply_json != {}: return assistant_reply_json - logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) + logger.error( + "Error: The following AI output couldn't be converted to a JSON:\n", + assistant_reply, + ) if CFG.speak_mode: say_text("I have received an invalid JSON response from the OpenAI API.") diff --git a/autogpt/json_utils/utilities.py b/autogpt/json_utils/utilities.py index 8499ddc853d7..eb9bb6877504 100644 --- a/autogpt/json_utils/utilities.py +++ b/autogpt/json_utils/utilities.py @@ -4,8 +4,8 @@ from jsonschema import Draft7Validator -from autogpt.logs import logger from autogpt.config import Config +from autogpt.logs import logger CFG = Config() diff --git a/autogpt/logs.py b/autogpt/logs.py index 50fe01282957..35037404a98f 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -204,7 +204,10 @@ def remove_color_codes(s: str) -> str: def print_assistant_thoughts(ai_name, assistant_reply): """Prints the assistant's thoughts to the console""" - from autogpt.json_utils.json_fix_llm import fix_and_parse_json, attempt_to_fix_json_by_finding_outermost_brackets + from autogpt.json_utils.json_fix_llm import ( + attempt_to_fix_json_by_finding_outermost_brackets, + fix_and_parse_json, + ) try: try: