From 4aae2fa16b6ea38ccbac15283646154d6bcd4864 Mon Sep 17 00:00:00 2001 From: Maxim Saplin Date: Fri, 22 Dec 2023 13:59:57 +0300 Subject: [PATCH 1/7] fix for #762 ,get_config_list/oia assetion, docs, test --- autogen/oai/openai_utils.py | 87 ++++++++++++++++++++++++++++++++----- test/oai/test_utils.py | 53 ++++++++++++++++++++++ 2 files changed, 130 insertions(+), 10 deletions(-) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 966646cdb802..2cbde16d0217 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -65,14 +65,36 @@ def get_key(config): def get_config_list( api_keys: List, base_urls: Optional[List] = None, api_type: Optional[str] = None, api_version: Optional[str] = None ) -> List[Dict]: - """Get a list of configs for openai api calls. + """Get a list of configs for OpenAI API client. Args: api_keys (list): The api keys for openai api calls. - base_urls (list, optional): The api bases for openai api calls. + base_urls (list, optional): The api bases for openai api calls. If provided, should match the length of api_keys. api_type (str, optional): The api type for openai api calls. api_version (str, optional): The api version for openai api calls. + + Returns: + list: A list of configs for OepnAI API calls. + + Example: + ``` + # Define a list of API keys + api_keys = ['key1', 'key2', 'key3'] + + # Optionally, define a list of base URLs corresponding to each API key + base_urls = ['https://api.service1.com', 'https://api.service2.com', 'https://api.service3.com'] + + # Optionally, define the API type and version if they are common for all keys + api_type = 'openai' + api_version = 'v1' + + # Call the get_config_list function to get a list of configuration dictionaries + config_list = get_config_list(api_keys, base_urls, api_type, api_version) + ``` + """ + if base_urls is not None: + assert len(api_keys) == len(base_urls), "The length of api_keys must match the length of base_urls" config_list = [] for i, api_key in enumerate(api_keys): if not api_key.strip(): @@ -92,20 +114,56 @@ def config_list_openai_aoai( key_file_path: Optional[str] = ".", openai_api_key_file: Optional[str] = "key_openai.txt", aoai_api_key_file: Optional[str] = "key_aoai.txt", + openai_api_base_file: Optional[str] = "base_openai.txt", aoai_api_base_file: Optional[str] = "base_aoai.txt", exclude: Optional[str] = None, ) -> List[Dict]: - """Get a list of configs for openai + azure openai api calls. + """Get a list of configs for OpenAI API client (including Azure or local model deployments that support OpenAI's chat completion API). + + This function constructs configurations by reading API keys and base URLs from environment variables or text files. + It supports configurations for both OpenAI and Azure OpenAI services, allowing for the exclusion of one or the other. Args: - key_file_path (str, optional): The path to the key files. - openai_api_key_file (str, optional): The file name of the openai api key. - aoai_api_key_file (str, optional): The file name of the azure openai api key. - aoai_api_base_file (str, optional): The file name of the azure openai api base. - exclude (str, optional): The api type to exclude, "openai" or "aoai". + key_file_path (str, optional): The directory path where the API key files are located. Defaults to the current directory. + openai_api_key_file (str, optional): The filename containing the OpenAI API key. Defaults to 'key_openai.txt'. + aoai_api_key_file (str, optional): The filename containing the Azure OpenAI API key. Defaults to 'key_aoai.txt'. + aoai_api_base_file (str, optional): The filename containing the Azure OpenAI API base URL. Defaults to 'base_aoai.txt'. + exclude (str, optional): The API type to exclude from the configuration list. Can be 'openai' or 'aoai'. Defaults to None. Returns: - list: A list of configs for openai api calls. + List[Dict]: A list of configuration dictionaries. Each dictionary contains keys for 'api_key', 'base_url', 'api_type', + and 'api_version'. + + Raises: + FileNotFoundError: If the specified key files are not found and the corresponding API key is not set in the environment variables. + + Example: + # To generate configurations excluding Azure OpenAI: + configs = config_list_openai_aoai(exclude='aoai') + + File samples: + - key_aoai.txt + + ``` + aoai-12345abcdef67890ghijklmnopqr + aoai-09876zyxwvuts54321fedcba + ``` + + - base_aoai.txt + + ``` + https://api.azure.com/v1 + https://api.azure2.com/v1 + ``` + + Notes: + - The function checks for API keys and base URLs in the following environment variables: 'OPENAI_API_KEY', 'AZURE_OPENAI_API_KEY', + 'OPENAI_API_BASE' and 'AZURE_OPENAI_API_BASE'. If these are not found, it attempts to read from the specified files in the + 'key_file_path' directory. + - The API version for Azure configurations is set to '2023-08-01-preview' by default and can be changed as necessary. + - If 'exclude' is set to 'openai', only Azure OpenAI configurations are returned, and vice versa. + - The function assumes that the API keys and base URLs in the environment variables are separated by new lines if there are + multiple entries. """ if "OPENAI_API_KEY" not in os.environ and exclude != "openai": try: @@ -116,6 +174,15 @@ def config_list_openai_aoai( "OPENAI_API_KEY is not found in os.environ " "and key_openai.txt is not found in the specified path. You can specify the api_key in the config_list." ) + if "OPENAI_API_BASE" not in os.environ and exclude != "openai": + try: + with open(f"{key_file_path}/{openai_api_base_file}") as key_file: + os.environ["OPENAI_API_BASE"] = key_file.read().strip() + except FileNotFoundError: + logging.info( + "OPENAI_API_BASE is not found in os.environ " + "and base_openai.txt is not found in the specified path. You can specify the base_url in the config_list." + ) if "AZURE_OPENAI_API_KEY" not in os.environ and exclude != "aoai": try: with open(f"{key_file_path}/{aoai_api_key_file}") as key_file: @@ -150,8 +217,8 @@ def config_list_openai_aoai( get_config_list( # Assuming OpenAI API_KEY in os.environ["OPENAI_API_KEY"] api_keys=os.environ.get("OPENAI_API_KEY", "").split("\n"), + base_urls=os.environ.get("OPENAI_API_BASE", "").split("\n"), # "api_type": "open_ai", - # "base_url": "https://api.openai.com/v1", ) if exclude != "openai" else [] diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py index 579fc6f9d8a2..294e8bccfa92 100644 --- a/test/oai/test_utils.py +++ b/test/oai/test_utils.py @@ -5,6 +5,7 @@ import logging import tempfile from unittest import mock +from unittest.mock import patch import autogen # noqa: E402 KEY_LOC = "notebook" @@ -77,6 +78,39 @@ def test_config_list_openai_aoai(): assert all(config.get("api_type") in [None, "open_ai", "azure"] for config in config_list) +@patch('os.environ', { + 'OPENAI_API_KEY': 'test_openai_key', + 'OPENAI_API_BASE': 'https://api.openai.com', + 'AZURE_OPENAI_API_KEY': 'test_aoai_key', + 'AZURE_OPENAI_API_BASE': 'https://api.azure.com' +}) +def test_config_list_openai_aoai_env_vars(): + # Test the config_list_openai_aoai function with environment variables set + configs = autogen.oai.openai_utils.config_list_openai_aoai() + assert len(configs) == 2 + assert {'api_key': 'test_openai_key', 'base_url': 'https://api.openai.com'} in configs + assert {'api_key': 'test_aoai_key', 'base_url': 'https://api.azure.com', 'api_type': 'azure', + 'api_version': '2023-08-01-preview'} in configs + + +@patch('os.environ', { + 'OPENAI_API_KEY': 'test_openai_key\ntest_openai_key2', + 'OPENAI_API_BASE': 'https://api.openai.com\nhttps://api.openai.com/v2', + 'AZURE_OPENAI_API_KEY': 'test_aoai_key\ntest_aoai_key2', + 'AZURE_OPENAI_API_BASE': 'https://api.azure.com\nhttps://api.azure.com/v2' +}) +def test_config_list_openai_aoai_env_vars_multi(): + # Test the config_list_openai_aoai function with multiple environment variable values (new line separated) + configs = autogen.oai.openai_utils.config_list_openai_aoai() + assert len(configs) == 4 + assert {'api_key': 'test_openai_key', 'base_url': 'https://api.openai.com'} in configs + assert {'api_key': 'test_openai_key2', 'base_url': 'https://api.openai.com/v2'} in configs + assert {'api_key': 'test_aoai_key', 'base_url': 'https://api.azure.com', 'api_type': 'azure', + 'api_version': '2023-08-01-preview'} in configs + assert {'api_key': 'test_aoai_key2', 'base_url': 'https://api.azure.com/v2', 'api_type': 'azure', + 'api_version': '2023-08-01-preview'} in configs + + def test_config_list_from_dotenv(mock_os_environ, caplog): # Test with valid .env file fd, temp_name = tempfile.mkstemp() @@ -160,5 +194,24 @@ def test_config_list_from_dotenv(mock_os_environ, caplog): assert "API key not found or empty for model gpt-4" in caplog.text +@patch('os.environ', { + 'OPENAI_API_KEY': 'test_openai_key\ntest_openai_key2', + 'OPENAI_API_BASE': 'https://api.openai.com', +}) +def test_api_keys_base_urls_length_mismatch(): + api_keys = ['key1', 'key2'] + base_urls = ['https://api.service1.com'] # Shorter than api_keys + + with pytest.raises(AssertionError) as exc_info: + autogen.get_config_list(api_keys, base_urls) + + assert str(exc_info.value) == "The length of api_keys must match the length of base_urls" + + with pytest.raises(AssertionError) as exc_info: + autogen.config_list_openai_aoai(api_keys, base_urls) + + assert str(exc_info.value) == "The length of api_keys must match the length of base_urls" + + if __name__ == "__main__": pytest.main() From a8b909ab8b878b2ecf7ad51fb200fd15dc18466e Mon Sep 17 00:00:00 2001 From: Maxim Saplin Date: Fri, 22 Dec 2023 16:11:22 +0300 Subject: [PATCH 2/7] Improved docstrings --- autogen/oai/openai_utils.py | 175 +++++++++++++++++++++++++++--------- 1 file changed, 132 insertions(+), 43 deletions(-) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 2cbde16d0217..f01706202ee9 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -235,18 +235,52 @@ def config_list_from_models( exclude: Optional[str] = None, model_list: Optional[list] = None, ) -> List[Dict]: - """Get a list of configs for api calls with models in the model list. + """ + Get a list of configs for API calls with models specified in the model list. + + This function extends `config_list_openai_aoai` by allowing to clone its' out for each fof the models provided. + Each configuration will have a 'model' key with the model name as its value. This is particularly useful when + all endpoints have same set of models. Args: key_file_path (str, optional): The path to the key files. - openai_api_key_file (str, optional): The file name of the openai api key. - aoai_api_key_file (str, optional): The file name of the azure openai api key. - aoai_api_base_file (str, optional): The file name of the azure openai api base. - exclude (str, optional): The api type to exclude, "openai" or "aoai". - model_list (list, optional): The model list. + openai_api_key_file (str, optional): The file name of the OpenAI API key. + aoai_api_key_file (str, optional): The file name of the Azure OpenAI API key. + aoai_api_base_file (str, optional): The file name of the Azure OpenAI API base. + exclude (str, optional): The API type to exclude, "openai" or "aoai". + model_list (list, optional): The list of model names to include in the configs. Returns: - list: A list of configs for openai api calls. + list: A list of configs for OpenAI API calls, each including model information. + + Example: + ``` + # Define the path where the API key files are located + key_file_path = '/path/to/key/files' + + # Define the file names for the OpenAI and Azure OpenAI API keys and bases + openai_api_key_file = 'key_openai.txt' + aoai_api_key_file = 'key_aoai.txt' + aoai_api_base_file = 'base_aoai.txt' + + # Define the list of models for which to create configurations + model_list = ['text-davinci-003', 'gpt-3.5-turbo'] + + # Call the function to get a list of configuration dictionaries + config_list = config_list_from_models( + key_file_path=key_file_path, + openai_api_key_file=openai_api_key_file, + aoai_api_key_file=aoai_api_key_file, + aoai_api_base_file=aoai_api_base_file, + model_list=model_list + ) + + # The `config_list` will contain configurations for the specified models, for example: + # [ + # {'api_key': '...', 'base_url': 'https://api.openai.com', 'model': 'text-davinci-003'}, + # {'api_key': '...', 'base_url': 'https://api.openai.com', 'model': 'gpt-3.5-turbo'} + # ] + ``` """ config_list = config_list_openai_aoai( key_file_path, @@ -267,7 +301,7 @@ def config_list_gpt4_gpt35( aoai_api_base_file: Optional[str] = "base_aoai.txt", exclude: Optional[str] = None, ) -> List[Dict]: - """Get a list of configs for gpt-4 followed by gpt-3.5 api calls. + """Get a list of configs for 'gpt-4' followed by 'gpt-3.5-turbo' API calls. Args: key_file_path (str, optional): The path to the key files. @@ -290,15 +324,50 @@ def config_list_gpt4_gpt35( def filter_config(config_list, filter_dict): - """Filter the config list by provider and model. + """ + This function filters `config_list` by checking each configuration dictionary against the + criteria specified in `filter_dict`. A configuration dictionary is retained if for every + key in `filter_dict`, see example below. Args: - config_list (list): The config list. - filter_dict (dict, optional): The filter dict with keys corresponding to a field in each config, - and values corresponding to lists of acceptable values for each key. + config_list (list of dict): A list of configuration dictionaries to be filtered. + filter_dict (dict): A dictionary representing the filter criteria, where each key is a + field name to check within the configuration dictionaries, and the + corresponding value is a list of acceptable values for that field. Returns: - list: The filtered config list. + list of dict: A list of configuration dictionaries that meet all the criteria specified + in `filter_dict`. + + Example: + ``` + # Example configuration list with various models and API types + configs = [ + {'model': 'gpt-3.5-turbo', 'api_type': 'openai'}, + {'model': 'gpt-4', 'api_type': 'openai'}, + {'model': 'gpt-3.5-turbo', 'api_type': 'azure'}, + ] + + # Define filter criteria to select configurations for the 'gpt-3.5-turbo' model + # that are also using the 'openai' API type + filter_criteria = { + 'model': ['gpt-3.5-turbo'], # Only accept configurations for 'gpt-3.5-turbo' + 'api_type': ['openai'] # Only accept configurations for 'openai' API type + } + + # Apply the filter to the configuration list + filtered_configs = filter_config(configs, filter_criteria) + + # The resulting `filtered_configs` will be: + # [{'model': 'gpt-3.5-turbo', 'api_type': 'openai'}] + ``` + + Note: + - If `filter_dict` is empty or None, no filtering is applied and `config_list` is returned as is. + - If a configuration dictionary in `config_list` does not contain a key specified in `filter_dict`, + it is considered a non-match and is excluded from the result. + - If the list of acceptable values for a key in `filter_dict` includes None, then configuration + dictionaries that do not have that key will also be considered a match. """ if filter_dict: config_list = [ @@ -312,23 +381,37 @@ def config_list_from_json( file_location: Optional[str] = "", filter_dict: Optional[Dict[str, Union[List[Union[str, None]], Set[Union[str, None]]]]] = None, ) -> List[Dict]: - """Get a list of configs from a json parsed from an env variable or a file. + """ + Retrieves a list of API configurations from a JSON stored in an environment variable or a file. + + This function attempts to parse JSON data from the given `env_or_file` parameter. If `env_or_file` is an + environment variable containing JSON data, it will be used directly. Otherwise, it is assumed to be a filename, + and the function will attempt to read the file from the specified `file_location`. + + The `filter_dict` parameter allows for filtering the configurations based on specified criteria. Each key in the + `filter_dict` corresponds to a field in the configuration dictionaries, and the associated value is a list or set + of acceptable values for that field. If a field is missing in a configuration and `None` is included in the list + of acceptable values for that field, the configuration will still be considered a match. Args: - env_or_file (str): The env variable name or file name. - file_location (str, optional): The file location. - filter_dict (dict, optional): The filter dict with keys corresponding to a field in each config, - and values corresponding to lists of acceptable values for each key. - e.g., - ```python - filter_dict = { - "api_type": ["open_ai", None], # None means a missing key is acceptable - "model": ["gpt-3.5-turbo", "gpt-4"], - } + env_or_file (str): The name of the environment variable or the filename containing the JSON data. + file_location (str, optional): The directory path where the file is located, if `env_or_file` is a filename. + filter_dict (dict, optional): A dictionary specifying the filtering criteria for the configurations, with + keys representing field names and values being lists or sets of acceptable values for those fields. + + Example: + ``` + # Suppose we have an environment variable 'CONFIG_JSON' with the following content: + # '[{"model": "gpt-3.5-turbo", "api_type": "openai"}, {"model": "gpt-4", "api_type": "openai"}]' + + # We can retrieve a filtered list of configurations like this: + filter_criteria = {"api_type": ["openai"], "model": ["gpt-3.5-turbo"]} + configs = config_list_from_json('CONFIG_JSON', filter_dict=filter_criteria) + # The 'configs' variable will now contain only the configurations that match the filter criteria. ``` Returns: - list: A list of configs for openai api calls. + List[Dict]: A list of configuration dictionaries that match the filtering criteria specified in `filter_dict`. """ json_str = os.environ.get(env_or_file) if json_str: @@ -348,27 +431,33 @@ def get_config( api_key: str, base_url: Optional[str] = None, api_type: Optional[str] = None, api_version: Optional[str] = None ) -> Dict: """ - Construct a configuration dictionary with the provided API configurations. - Appending the additional configurations to the config only if they're set - - example: - >> model_api_key_map={ - "gpt-4": "OPENAI_API_KEY", - "gpt-3.5-turbo": { - "api_key_env_var": "ANOTHER_API_KEY", - "api_type": "aoai", - "api_version": "v2", - "base_url": "https://api.someotherapi.com" - } - } + Constructs a configuration dictionary for a single model with the provided API configurations. + + Example: + ``` + config = get_config( + api_key="sk-abcdef1234567890", + base_url="https://api.openai.com", + api_type="openai", + api_version="v1" + ) + # The 'config' variable will now contain: + # { + # "api_key": "sk-abcdef1234567890", + # "base_url": "https://api.openai.com", + # "api_type": "openai", + # "api_version": "v1" + # } + ``` + Args: - api_key (str): The API key used for authenticating API requests. - base_url (str, optional): The base URL of the API. Defaults to None. - api_type (str, optional): The type or kind of API. Defaults to None. - api_version (str, optional): The API version. Defaults to None. + api_key (str): The API key for authenticating API requests. + base_url (Optional[str]): The base URL of the API. If not provided, defaults to None. + api_type (Optional[str]): The type of API. If not provided, defaults to None. + api_version (Optional[str]): The version of the API. If not provided, defaults to None. Returns: - Dict: A dictionary containing the API configurations. + Dict: A dictionary containing the provided API configurations. """ config = {"api_key": api_key} if base_url: From 244f6326852b27ad49b6ef88c5bb58e8a5fc8f8e Mon Sep 17 00:00:00 2001 From: Maxim Saplin Date: Fri, 22 Dec 2023 17:32:29 +0300 Subject: [PATCH 3/7] test_utils using fake data and temp files --- test/oai/test_client_stream.py | 4 +- test/oai/test_utils.py | 95 ++++++++++++++++++++++++++-------- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/test/oai/test_client_stream.py b/test/oai/test_client_stream.py index 2583c4cac2b6..620a02933356 100644 --- a/test/oai/test_client_stream.py +++ b/test/oai/test_client_stream.py @@ -1,6 +1,5 @@ import pytest from autogen import OpenAIWrapper, config_list_from_json, config_list_openai_aoai -from test_utils import OAI_CONFIG_LIST, KEY_LOC try: from openai import OpenAI @@ -9,6 +8,9 @@ else: skip = False +KEY_LOC = "notebook" +OAI_CONFIG_LIST = "OAI_CONFIG_LIST" + @pytest.mark.skipif(skip, reason="openai>=1 not installed") def test_aoai_chat_completion_stream(): diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py index 294e8bccfa92..6c1daac00356 100644 --- a/test/oai/test_utils.py +++ b/test/oai/test_utils.py @@ -8,9 +8,6 @@ from unittest.mock import patch import autogen # noqa: E402 -KEY_LOC = "notebook" -OAI_CONFIG_LIST = "OAI_CONFIG_LIST" - sys.path.append("../../autogen") # Example environment variables @@ -39,6 +36,31 @@ } } +JSON_SAMPLE = ''' +[ + { + "model": "gpt-3.5-turbo", + "api_type": "openai" + }, + { + "model": "gpt-4", + "api_type": "openai" + }, + { + "model": "gpt-35-turbo-v0301", + "api_key": "111113fc7e8a46419bfac511bb301111", + "base_url": "https://1111.openai.azure.com", + "api_type": "azure", + "api_version": "2023-07-01-preview" + }, + { + "model": "gpt", + "api_key": "not-needed", + "base_url": "http://localhost:1234/v1" + } +] +''' + @pytest.fixture def mock_os_environ(): @@ -47,35 +69,64 @@ def mock_os_environ(): def test_config_list_from_json(): - # Test the functionality for loading configurations from JSON file - # and ensuring that the loaded configurations are as expected. - config_list = autogen.config_list_gpt4_gpt35(key_file_path=KEY_LOC) - json_file = os.path.join(KEY_LOC, "config_list_test.json") + with tempfile.NamedTemporaryFile(mode='w+', delete=False) as tmp_file: + json_data = json.loads(JSON_SAMPLE) + tmp_file.write(JSON_SAMPLE) + tmp_file.flush() - with open(json_file, "w") as f: - json.dump(config_list, f, indent=4) + config_list = autogen.config_list_from_json(tmp_file.name) - config_list_1 = autogen.config_list_from_json(json_file) - assert config_list == config_list_1 + assert len(config_list) == len(json_data) + i = 0 + for config in config_list: + assert isinstance(config, dict) + for key in config: + assert key in json_data[i] + assert config[key] == json_data[i][key] + i += 1 - os.environ["config_list_test"] = json.dumps(config_list) - config_list_2 = autogen.config_list_from_json("config_list_test") - assert config_list == config_list_2 + os.environ["config_list_test"] = JSON_SAMPLE + config_list_2 = autogen.config_list_from_json("config_list_test") + assert config_list == config_list_2 - config_list_3 = autogen.config_list_from_json( - OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"model": ["gpt4", "gpt-4-32k"]} - ) - assert all(config.get("model") in ["gpt4", "gpt-4-32k"] for config in config_list_3) + config_list_3 = autogen.config_list_from_json( + tmp_file.name, filter_dict={"model": ["gpt", "gpt-4", "gpt-4-32k"]} + ) + assert all(config.get("model") in ["gpt-4", "gpt"] for config in config_list_3) - del os.environ["config_list_test"] - os.remove(json_file) + del os.environ["config_list_test"] def test_config_list_openai_aoai(): # Testing the functionality for loading configurations for different API types # and ensuring the API types in the loaded configurations are as expected. - config_list = autogen.config_list_openai_aoai(key_file_path=KEY_LOC) - assert all(config.get("api_type") in [None, "open_ai", "azure"] for config in config_list) + with tempfile.TemporaryDirectory() as temp_dir: + # Create temporary files with sample data for keys and base URLs + openai_key_file = os.path.join(temp_dir, "key_openai.txt") + aoai_key_file = os.path.join(temp_dir, "key_aoai.txt") + openai_base_file = os.path.join(temp_dir, "base_openai.txt") + aoai_base_file = os.path.join(temp_dir, "base_aoai.txt") + + # Write sample data to the temporary files + with open(openai_key_file, 'w') as f: + f.write("sk-testkeyopenai123\nsk-testkeyopenai456") + with open(aoai_key_file, 'w') as f: + f.write("sk-testkeyaoai456") + with open(openai_base_file, 'w') as f: + f.write("https://api.openai.com/v1\nhttps://api.openai.com/v1") + with open(aoai_base_file, 'w') as f: + f.write("https://api.azure.com/v1") + + # Pass the temporary directory as a parameter to the function + config_list = autogen.config_list_openai_aoai(key_file_path=temp_dir) + assert len(config_list) == 3 + expected_config_list = [ + {'api_key': 'sk-testkeyopenai123', 'base_url': 'https://api.openai.com/v1'}, + {'api_key': 'sk-testkeyopenai456', 'base_url': 'https://api.openai.com/v1'}, + {'api_key': 'sk-testkeyaoai456', 'base_url': 'https://api.azure.com/v1', 'api_type': 'azure', + 'api_version': '2023-08-01-preview'} + ] + assert config_list == expected_config_list @patch('os.environ', { From 613441143cf4ce6d4a8653f4efd07bf828b1aefe Mon Sep 17 00:00:00 2001 From: Maxim Saplin Date: Sun, 24 Dec 2023 15:13:28 +0300 Subject: [PATCH 4/7] "Black" formatting applied --- test/oai/test_utils.py | 101 +++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py index 6c1daac00356..b9817781c777 100644 --- a/test/oai/test_utils.py +++ b/test/oai/test_utils.py @@ -36,7 +36,7 @@ } } -JSON_SAMPLE = ''' +JSON_SAMPLE = """ [ { "model": "gpt-3.5-turbo", @@ -59,7 +59,7 @@ "base_url": "http://localhost:1234/v1" } ] -''' +""" @pytest.fixture @@ -69,7 +69,7 @@ def mock_os_environ(): def test_config_list_from_json(): - with tempfile.NamedTemporaryFile(mode='w+', delete=False) as tmp_file: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp_file: json_data = json.loads(JSON_SAMPLE) tmp_file.write(JSON_SAMPLE) tmp_file.flush() @@ -108,58 +108,80 @@ def test_config_list_openai_aoai(): aoai_base_file = os.path.join(temp_dir, "base_aoai.txt") # Write sample data to the temporary files - with open(openai_key_file, 'w') as f: + with open(openai_key_file, "w") as f: f.write("sk-testkeyopenai123\nsk-testkeyopenai456") - with open(aoai_key_file, 'w') as f: + with open(aoai_key_file, "w") as f: f.write("sk-testkeyaoai456") - with open(openai_base_file, 'w') as f: + with open(openai_base_file, "w") as f: f.write("https://api.openai.com/v1\nhttps://api.openai.com/v1") - with open(aoai_base_file, 'w') as f: + with open(aoai_base_file, "w") as f: f.write("https://api.azure.com/v1") # Pass the temporary directory as a parameter to the function config_list = autogen.config_list_openai_aoai(key_file_path=temp_dir) assert len(config_list) == 3 expected_config_list = [ - {'api_key': 'sk-testkeyopenai123', 'base_url': 'https://api.openai.com/v1'}, - {'api_key': 'sk-testkeyopenai456', 'base_url': 'https://api.openai.com/v1'}, - {'api_key': 'sk-testkeyaoai456', 'base_url': 'https://api.azure.com/v1', 'api_type': 'azure', - 'api_version': '2023-08-01-preview'} + {"api_key": "sk-testkeyopenai123", "base_url": "https://api.openai.com/v1"}, + {"api_key": "sk-testkeyopenai456", "base_url": "https://api.openai.com/v1"}, + { + "api_key": "sk-testkeyaoai456", + "base_url": "https://api.azure.com/v1", + "api_type": "azure", + "api_version": "2023-08-01-preview", + }, ] assert config_list == expected_config_list -@patch('os.environ', { - 'OPENAI_API_KEY': 'test_openai_key', - 'OPENAI_API_BASE': 'https://api.openai.com', - 'AZURE_OPENAI_API_KEY': 'test_aoai_key', - 'AZURE_OPENAI_API_BASE': 'https://api.azure.com' -}) +@patch( + "os.environ", + { + "OPENAI_API_KEY": "test_openai_key", + "OPENAI_API_BASE": "https://api.openai.com", + "AZURE_OPENAI_API_KEY": "test_aoai_key", + "AZURE_OPENAI_API_BASE": "https://api.azure.com", + }, +) def test_config_list_openai_aoai_env_vars(): # Test the config_list_openai_aoai function with environment variables set configs = autogen.oai.openai_utils.config_list_openai_aoai() assert len(configs) == 2 - assert {'api_key': 'test_openai_key', 'base_url': 'https://api.openai.com'} in configs - assert {'api_key': 'test_aoai_key', 'base_url': 'https://api.azure.com', 'api_type': 'azure', - 'api_version': '2023-08-01-preview'} in configs + assert {"api_key": "test_openai_key", "base_url": "https://api.openai.com"} in configs + assert { + "api_key": "test_aoai_key", + "base_url": "https://api.azure.com", + "api_type": "azure", + "api_version": "2023-08-01-preview", + } in configs -@patch('os.environ', { - 'OPENAI_API_KEY': 'test_openai_key\ntest_openai_key2', - 'OPENAI_API_BASE': 'https://api.openai.com\nhttps://api.openai.com/v2', - 'AZURE_OPENAI_API_KEY': 'test_aoai_key\ntest_aoai_key2', - 'AZURE_OPENAI_API_BASE': 'https://api.azure.com\nhttps://api.azure.com/v2' -}) +@patch( + "os.environ", + { + "OPENAI_API_KEY": "test_openai_key\ntest_openai_key2", + "OPENAI_API_BASE": "https://api.openai.com\nhttps://api.openai.com/v2", + "AZURE_OPENAI_API_KEY": "test_aoai_key\ntest_aoai_key2", + "AZURE_OPENAI_API_BASE": "https://api.azure.com\nhttps://api.azure.com/v2", + }, +) def test_config_list_openai_aoai_env_vars_multi(): # Test the config_list_openai_aoai function with multiple environment variable values (new line separated) configs = autogen.oai.openai_utils.config_list_openai_aoai() assert len(configs) == 4 - assert {'api_key': 'test_openai_key', 'base_url': 'https://api.openai.com'} in configs - assert {'api_key': 'test_openai_key2', 'base_url': 'https://api.openai.com/v2'} in configs - assert {'api_key': 'test_aoai_key', 'base_url': 'https://api.azure.com', 'api_type': 'azure', - 'api_version': '2023-08-01-preview'} in configs - assert {'api_key': 'test_aoai_key2', 'base_url': 'https://api.azure.com/v2', 'api_type': 'azure', - 'api_version': '2023-08-01-preview'} in configs + assert {"api_key": "test_openai_key", "base_url": "https://api.openai.com"} in configs + assert {"api_key": "test_openai_key2", "base_url": "https://api.openai.com/v2"} in configs + assert { + "api_key": "test_aoai_key", + "base_url": "https://api.azure.com", + "api_type": "azure", + "api_version": "2023-08-01-preview", + } in configs + assert { + "api_key": "test_aoai_key2", + "base_url": "https://api.azure.com/v2", + "api_type": "azure", + "api_version": "2023-08-01-preview", + } in configs def test_config_list_from_dotenv(mock_os_environ, caplog): @@ -245,13 +267,16 @@ def test_config_list_from_dotenv(mock_os_environ, caplog): assert "API key not found or empty for model gpt-4" in caplog.text -@patch('os.environ', { - 'OPENAI_API_KEY': 'test_openai_key\ntest_openai_key2', - 'OPENAI_API_BASE': 'https://api.openai.com', -}) +@patch( + "os.environ", + { + "OPENAI_API_KEY": "test_openai_key\ntest_openai_key2", + "OPENAI_API_BASE": "https://api.openai.com", + }, +) def test_api_keys_base_urls_length_mismatch(): - api_keys = ['key1', 'key2'] - base_urls = ['https://api.service1.com'] # Shorter than api_keys + api_keys = ["key1", "key2"] + base_urls = ["https://api.service1.com"] # Shorter than api_keys with pytest.raises(AssertionError) as exc_info: autogen.get_config_list(api_keys, base_urls) From b472731acf7ec7cd07ab423e80aec215aa32e9c3 Mon Sep 17 00:00:00 2001 From: Maxim Saplin Date: Sun, 24 Dec 2023 17:04:58 +0300 Subject: [PATCH 5/7] Fix build (KEY_LOC and OAI_CONFIG_LIST as consts in test_client) --- test/oai/test_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/oai/test_client.py b/test/oai/test_client.py index aec241697ec3..14a7373cf860 100644 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -1,6 +1,5 @@ import pytest from autogen import OpenAIWrapper, config_list_from_json, config_list_openai_aoai -from test_utils import OAI_CONFIG_LIST, KEY_LOC TOOL_ENABLED = False try: @@ -15,6 +14,9 @@ if openai.__version__ >= "1.1.0": TOOL_ENABLED = True +KEY_LOC = "notebook" +OAI_CONFIG_LIST = "OAI_CONFIG_LIST" + @pytest.mark.skipif(skip, reason="openai>=1 not installed") def test_aoai_chat_completion(): From 7994367690979ffef93a8a2ba9af1bf0dc17e059 Mon Sep 17 00:00:00 2001 From: Maxim Saplin Date: Thu, 28 Dec 2023 20:56:42 +0300 Subject: [PATCH 6/7] Ramping up openai_utils coverage --- test/oai/test_utils.py | 59 ++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py index b9817781c777..aca1556db059 100644 --- a/test/oai/test_utils.py +++ b/test/oai/test_utils.py @@ -184,6 +184,12 @@ def test_config_list_openai_aoai_env_vars_multi(): } in configs +def test_config_list_openai_aoai_file_not_found(): + with mock.patch.dict(os.environ, {}, clear=True): + config_list = autogen.config_list_openai_aoai(key_file_path="non_existent_path") + assert len(config_list) == 0 + + def test_config_list_from_dotenv(mock_os_environ, caplog): # Test with valid .env file fd, temp_name = tempfile.mkstemp() @@ -267,27 +273,52 @@ def test_config_list_from_dotenv(mock_os_environ, caplog): assert "API key not found or empty for model gpt-4" in caplog.text -@patch( - "os.environ", - { - "OPENAI_API_KEY": "test_openai_key\ntest_openai_key2", - "OPENAI_API_BASE": "https://api.openai.com", - }, -) -def test_api_keys_base_urls_length_mismatch(): - api_keys = ["key1", "key2"] - base_urls = ["https://api.service1.com"] # Shorter than api_keys +def test_get_config_list(): + # Define a list of API keys and corresponding base URLs + api_keys = ["key1", "key2", "key3"] + base_urls = ["https://api.service1.com", "https://api.service2.com", "https://api.service3.com"] + api_type = "openai" + api_version = "v1" - with pytest.raises(AssertionError) as exc_info: - autogen.get_config_list(api_keys, base_urls) + # Call the get_config_list function to get a list of configuration dictionaries + config_list = autogen.get_config_list(api_keys, base_urls, api_type, api_version) - assert str(exc_info.value) == "The length of api_keys must match the length of base_urls" + # Check that the config_list is not empty + assert config_list, "The config_list should not be empty." + + # Check that the config_list has the correct length + assert len(config_list) == len( + api_keys + ), "The config_list should have the same number of items as the api_keys list." + + # Check that each config in the config_list has the correct structure and data + for i, config in enumerate(config_list): + assert config["api_key"] == api_keys[i], f"The api_key for config {i} is incorrect." + assert config["base_url"] == base_urls[i], f"The base_url for config {i} is incorrect." + assert config["api_type"] == api_type, f"The api_type for config {i} is incorrect." + assert config["api_version"] == api_version, f"The api_version for config {i} is incorrect." + # Test with mismatched lengths of api_keys and base_urls with pytest.raises(AssertionError) as exc_info: - autogen.config_list_openai_aoai(api_keys, base_urls) + autogen.get_config_list(api_keys, base_urls[:2], api_type, api_version) + assert str(exc_info.value) == "The length of api_keys must match the length of base_urls" + # Test with empty api_keys + with pytest.raises(AssertionError) as exc_info: + autogen.get_config_list([], base_urls, api_type, api_version) assert str(exc_info.value) == "The length of api_keys must match the length of base_urls" + # Test with None base_urls + config_list_without_base = autogen.get_config_list(api_keys, None, api_type, api_version) + assert all( + "base_url" not in config for config in config_list_without_base + ), "The configs should not have base_url when None is provided." + + # Test with empty string in api_keys + api_keys_with_empty = ["key1", "", "key3"] + config_list_with_empty_key = autogen.get_config_list(api_keys_with_empty, base_urls, api_type, api_version) + assert len(config_list_with_empty_key) == 2, "The config_list should exclude configurations with empty api_keys." + if __name__ == "__main__": pytest.main() From 3ce4ba228d96f324dfed066f17074f21dea8dceb Mon Sep 17 00:00:00 2001 From: Eric Zhu Date: Thu, 28 Dec 2023 20:41:33 -0800 Subject: [PATCH 7/7] Missing parameter doc. --- autogen/oai/openai_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 351f0bea3fda..1390faed6b22 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -136,6 +136,7 @@ def config_list_openai_aoai( key_file_path (str, optional): The directory path where the API key files are located. Defaults to the current directory. openai_api_key_file (str, optional): The filename containing the OpenAI API key. Defaults to 'key_openai.txt'. aoai_api_key_file (str, optional): The filename containing the Azure OpenAI API key. Defaults to 'key_aoai.txt'. + openai_api_base_file (str, optional): The filename containing the OpenAI API base URL. Defaults to 'base_openai.txt'. aoai_api_base_file (str, optional): The filename containing the Azure OpenAI API base URL. Defaults to 'base_aoai.txt'. exclude (str, optional): The API type to exclude from the configuration list. Can be 'openai' or 'aoai'. Defaults to None.