From 0c490a9ec263c63938b2c6a5dedfdac26c90ab35 Mon Sep 17 00:00:00 2001 From: Erick Galinkin Date: Tue, 13 Feb 2024 16:40:10 -0500 Subject: [PATCH 1/2] Update gitignore. Upgrade openai to version 1.12.0 and refactor OpenAIGenerator to handle new calling conventions. Add test_openai.py. --- .gitignore | 1 + garak/generators/openai.py | 64 +++++++++++++++++++++------------ requirements.txt | 2 +- tests/generators/test_openai.py | 42 ++++++++++++++++++++++ 4 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 tests/generators/test_openai.py diff --git a/.gitignore b/.gitignore index 8c8f7a85d..09f5bac95 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,4 @@ garak.log hitlog.*.jsonl .vscode runs/ +logs/ \ No newline at end of file diff --git a/garak/generators/openai.py b/garak/generators/openai.py index ed6cd8aae..391cf018a 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -13,7 +13,8 @@ import os import re -from typing import List +import logging +from typing import List, Union import openai import backoff @@ -39,10 +40,7 @@ "gpt-4", "gpt-4-32k", "gpt-3.5-turbo", - "gpt-3.5-turbo-1106", "gpt-3.5-turbo-16k", - "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-16k-0613", "gpt-3.5-turbo-0301", ) @@ -65,24 +63,28 @@ def __init__(self, name, generations=10): super().__init__(name, generations=generations) - openai.api_key = os.getenv("OPENAI_API_KEY", default=None) - if openai.api_key is None: + api_key = os.getenv("OPENAI_API_KEY", default=None) + if api_key is None: raise ValueError( 'Put the OpenAI API key in the OPENAI_API_KEY environment variable (this was empty)\n \ e.g.: export OPENAI_API_KEY="sk-123XXXXXXXXXXXX"' ) + self.client = openai.OpenAI( + api_key=api_key + ) + if self.name in completion_models: - self.generator = openai.Completion + self.generator = self.client.completions elif self.name in chat_models: - self.generator = openai.ChatCompletion + self.generator = self.client.chat.completions elif "-".join(self.name.split("-")[:-1]) in chat_models and re.match( r"^.+-[01][0-9][0-3][0-9]$", self.name ): # handle model names -MMDDish suffix - self.generator = openai.ChatCompletion + self.generator = self.client.completions elif self.name == "": - openai_model_list = sorted([m["id"] for m in openai.Model().list()["data"]]) + openai_model_list = sorted([m.id for m in self.client.models.list().data]) raise ValueError( "Model name is required for OpenAI, use --model_name\n" + " API returns following available models: ▶️ " @@ -95,19 +97,27 @@ def __init__(self, name, generations=10): f"No OpenAI API defined for '{self.name}' in generators/openai.py - please add one!" ) + # noinspection PyArgumentList @backoff.on_exception( backoff.fibo, ( - openai.error.RateLimitError, - openai.error.ServiceUnavailableError, - openai.error.APIError, - openai.error.Timeout, - openai.error.APIConnectionError, + openai.RateLimitError, + openai.InternalServerError, + openai.APIError, + openai.APITimeoutError, + openai.APIConnectionError, ), max_value=70, ) - def _call_model(self, prompt: str) -> List[str]: - if self.generator == openai.Completion: + def _call_model(self, prompt: Union[str, list[dict]]) -> List[str]: + if self.generator == self.client.completions: + if not isinstance(prompt, str): + msg = (f"Expected a string for OpenAI completions model {self.name}, but got {type(prompt)}. " + f"Returning nothing!") + logging.error(msg) + print(msg) + return list() + response = self.generator.create( model=self.name, prompt=prompt, @@ -119,12 +129,22 @@ def _call_model(self, prompt: str) -> List[str]: presence_penalty=self.presence_penalty, stop=self.stop, ) - return [c["text"] for c in response["choices"]] - - elif self.generator == openai.ChatCompletion: + return [c.text for c in response.choices] + + elif self.generator == self.client.chat.completions: + if isinstance(prompt, str): + messages = [{"role": "user", "content": prompt}] + elif isinstance(prompt, list): + messages = prompt + else: + msg = (f"Expected a list of dicts for OpenAI Chat model {self.name}, but got {type(prompt)} instead. " + f"Returning nothing!") + logging.error(msg) + print(msg) + return list() response = self.generator.create( model=self.name, - messages=[{"role": "user", "content": prompt}], + messages=messages, temperature=self.temperature, top_p=self.top_p, n=self.generations, @@ -133,7 +153,7 @@ def _call_model(self, prompt: str) -> List[str]: presence_penalty=self.presence_penalty, frequency_penalty=self.frequency_penalty, ) - return [c["message"]["content"] for c in response["choices"]] + return [c.message.content for c in response.choices] else: raise ValueError( diff --git a/requirements.txt b/requirements.txt index acf106157..37e20a947 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ datasets>=2 colorama>=0.4.3 tqdm>=4.64.0 cohere>=4.5.1 -openai>=0.27.7,<1.0.0 +openai==1.12.0 replicate>=0.8.3 pytest>=7.3 google-api-python-client>=2.0 diff --git a/tests/generators/test_openai.py b/tests/generators/test_openai.py new file mode 100644 index 000000000..3a7832375 --- /dev/null +++ b/tests/generators/test_openai.py @@ -0,0 +1,42 @@ +from garak.generators.openai import OpenAIGenerator + +DEFAULT_GENERATIONS_QTY = 10 + + +def test_completion(): + generator = OpenAIGenerator(name="gpt-3.5-turbo-instruct") + assert generator.name == "gpt-3.5-turbo-instruct" + assert generator.generations == DEFAULT_GENERATIONS_QTY + assert isinstance(generator.max_tokens, int) + generator.max_tokens = 99 + assert generator.max_tokens == 99 + generator.temperature = 0.5 + assert generator.temperature == 0.5 + output = generator.generate("How could I possibly ") + assert len(output) == DEFAULT_GENERATIONS_QTY + for item in output: + assert isinstance(item, str) + print("test passed!") + + +def test_chat(): + generator = OpenAIGenerator(name="gpt-3.5-turbo") + assert generator.name == "gpt-3.5-turbo" + assert generator.generations == DEFAULT_GENERATIONS_QTY + assert isinstance(generator.max_tokens, int) + generator.max_tokens = 99 + assert generator.max_tokens == 99 + generator.temperature = 0.5 + assert generator.temperature == 0.5 + output = generator.generate("Hello OpenAI!") + assert len(output) == DEFAULT_GENERATIONS_QTY + for item in output: + assert isinstance(item, str) + messages = [{"role": "user", "content": "Hello OpenAI!"}, + {"role": "assistant", "content": "Hello! How can I help you today?"}, + {"role": "user", "content": "How do I write a sonnet?"}] + output = generator.generate(messages) + assert len(output) == DEFAULT_GENERATIONS_QTY + for item in output: + assert isinstance(item, str) + print("test passed!") From db1c7a6cb7d0a1f7adc6f204728e87310ae4e678 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Thu, 15 Feb 2024 16:07:28 +0100 Subject: [PATCH 2/2] black, update pyproject.toml --- garak/generators/openai.py | 16 +++++++++------- garak/probes/dan.py | 1 - garak/resources/promptinject/prompt_data.py | 1 + garak/resources/promptinject/scoring.py | 1 + pyproject.toml | 2 +- tests/generators/test_openai.py | 8 +++++--- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/garak/generators/openai.py b/garak/generators/openai.py index 391cf018a..a357ef8f5 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -70,9 +70,7 @@ def __init__(self, name, generations=10): e.g.: export OPENAI_API_KEY="sk-123XXXXXXXXXXXX"' ) - self.client = openai.OpenAI( - api_key=api_key - ) + self.client = openai.OpenAI(api_key=api_key) if self.name in completion_models: self.generator = self.client.completions @@ -112,8 +110,10 @@ def __init__(self, name, generations=10): def _call_model(self, prompt: Union[str, list[dict]]) -> List[str]: if self.generator == self.client.completions: if not isinstance(prompt, str): - msg = (f"Expected a string for OpenAI completions model {self.name}, but got {type(prompt)}. " - f"Returning nothing!") + msg = ( + f"Expected a string for OpenAI completions model {self.name}, but got {type(prompt)}. " + f"Returning nothing!" + ) logging.error(msg) print(msg) return list() @@ -137,8 +137,10 @@ def _call_model(self, prompt: Union[str, list[dict]]) -> List[str]: elif isinstance(prompt, list): messages = prompt else: - msg = (f"Expected a list of dicts for OpenAI Chat model {self.name}, but got {type(prompt)} instead. " - f"Returning nothing!") + msg = ( + f"Expected a list of dicts for OpenAI Chat model {self.name}, but got {type(prompt)} instead. " + f"Returning nothing!" + ) logging.error(msg) print(msg) return list() diff --git a/garak/probes/dan.py b/garak/probes/dan.py index 1a0c6a32b..ddebbe765 100644 --- a/garak/probes/dan.py +++ b/garak/probes/dan.py @@ -484,7 +484,6 @@ def __init__( class DanInTheWild(Probe): - """Tries a set of ~700 known jailbreaks, in an attempt to get a model to disregard alignment/system prompt. Success is measured by not getting a safety mitigaton message.""" active = False diff --git a/garak/resources/promptinject/prompt_data.py b/garak/resources/promptinject/prompt_data.py index 429712e0a..a0bd26139 100644 --- a/garak/resources/promptinject/prompt_data.py +++ b/garak/resources/promptinject/prompt_data.py @@ -1,6 +1,7 @@ """ Column aliases generated by the prompt. """ + column_aliases = { "prompt_secret_instruction": "Secret Instruction", "prompt_private_value": "Private Value", diff --git a/garak/resources/promptinject/scoring.py b/garak/resources/promptinject/scoring.py index a65cf0090..a48f42b59 100644 --- a/garak/resources/promptinject/scoring.py +++ b/garak/resources/promptinject/scoring.py @@ -4,6 +4,7 @@ TODO: - Use custom exceptions """ + import rapidfuzz from .prompt_data import column_aliases diff --git a/pyproject.toml b/pyproject.toml index edbf4dbe5..c6d71ad85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "colorama>=0.4.3", "tqdm>=4.64.0", "cohere>=4.5.1", - "openai>=0.27.7,<1.0.0", + "openai==1.12.0", "replicate>=0.8.3", "pytest>=7.3", "google-api-python-client>=2.0", diff --git a/tests/generators/test_openai.py b/tests/generators/test_openai.py index 3a7832375..e53d845e2 100644 --- a/tests/generators/test_openai.py +++ b/tests/generators/test_openai.py @@ -32,9 +32,11 @@ def test_chat(): assert len(output) == DEFAULT_GENERATIONS_QTY for item in output: assert isinstance(item, str) - messages = [{"role": "user", "content": "Hello OpenAI!"}, - {"role": "assistant", "content": "Hello! How can I help you today?"}, - {"role": "user", "content": "How do I write a sonnet?"}] + messages = [ + {"role": "user", "content": "Hello OpenAI!"}, + {"role": "assistant", "content": "Hello! How can I help you today?"}, + {"role": "user", "content": "How do I write a sonnet?"}, + ] output = generator.generate(messages) assert len(output) == DEFAULT_GENERATIONS_QTY for item in output: