From f383412addd8086a51051d06a47b6232a112d2b5 Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Wed, 4 Sep 2024 05:22:10 -0700 Subject: [PATCH 01/14] add support for groq API --- README.md | 6 +++ garak/generators/groq.py | 88 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 garak/generators/groq.py diff --git a/README.md b/README.md index 9b565ba8..03f3ce7a 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,12 @@ Private Replicate endpoints: * `--model_name` (optional, `command` by default) - The specific Cohere model you'd like to test * set the `COHERE_API_KEY` environment variable to your Cohere API key, e.g. "aBcDeFgHiJ123456789"; see https://dashboard.cohere.ai/api-keys when logged in +### Groq + +* `--model_type groq` +* `--model_name` - The name of the model to access via the Groq API +* set the `GROQ_API_KEY` environment variable to your Groq API key, see https://console.groq.com/docs/quickstart for details on creating an API key + ### ggml * `--model_type ggml` diff --git a/garak/generators/groq.py b/garak/generators/groq.py new file mode 100644 index 00000000..a6364b70 --- /dev/null +++ b/garak/generators/groq.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""NVIDIA Inference Microservice LLM interface""" + +import random +from typing import List, Union + +import openai + +from garak.generators.openai import OpenAICompatible + + +class GroqChat(OpenAICompatible): + """Wrapper for Groq-hosted LLM models. Expects GROQ_API_KEY environment variable. + + Uses the [OpenAI-compatible API](https://console.groq.com/docs/openai) + + To get started with this generator: + #. Visit [https://build.nvidia.com/explore/reasoning](build.nvidia.com/explore/reasoning) + and find the LLM you'd like to use. + #. On the page for the LLM you want to use (e.g. [mixtral-8x7b-instruct](https://build.nvidia.com/mistralai/mixtral-8x7b-instruct)), + click "Get API key" key above the code snippet. You may need to create an + account. Copy this key. + #. In your console, Set the ``NIM_API_KEY`` variable to this API key. On + Linux, this might look like ``export NIM_API_KEY="nvapi-xXxXxXx"``. + #. Run garak, setting ``--model_name`` to ``nim`` and ``--model_type`` to + the name of the model on [build.nvidia.com](https://build.nvidia.com/) + - e.g. ``mistralai/mixtral-8x7b-instruct-v0.1``. + + """ + + # per https://console.groq.com/docs/openai + # 2024.09.04, `n>1` is not supported + ENV_VAR = "GROQ_API_KEY" + DEFAULT_PARAMS = OpenAICompatible.DEFAULT_PARAMS | { + "temperature": 0.7, + "top_p": 1.0, + "uri": "https://api.groq.com/openai/v1", + "vary_seed_each_call": True, # encourage variation when generations>1 + "vary_temp_each_call": True, # encourage variation when generations>1 + "suppressed_params": { + "n", + "frequency_penalty", + "presence_penalty", + "logprobs", + "logit_bias", + "top_logprobs", + }, + } + active = True + supports_multiple_generations = False + generator_family_name = "Groq" + + timeout = 60 + + def _load_client(self): + self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key) + if self.name in ("", None): + raise ValueError( + "Groq API requires model name to be set, e.g. --model_name llama-3.1-8b-instant \nCurrent models:\n" + + "\n - ".join( + sorted([entry.id for entry in self.client.models.list().data]) + ) + ) + self.generator = self.client.chat.completions + + def _clear_client(self): + self.generator = None + self.client = None + + def _call_model( + self, prompt: str | List[dict], generations_this_call: int = 1 + ) -> List[Union[str, None]]: + assert ( + generations_this_call == 1 + ), "generations_per_call / n > 1 is not supported" + + if self.vary_seed_each_call: + self.seed = random.randint(0, 65535) + + if self.vary_temp_each_call: + self.temperature = random.random() + + return super()._call_model(prompt, generations_this_call) + + +DEFAULT_CLASS = "GroqChat" From 08e062be6f856a6ee71c79b50b2aabd0f003f67f Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Wed, 4 Sep 2024 05:25:52 -0700 Subject: [PATCH 02/14] update docs --- garak/generators/groq.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index a6364b70..2b9c4a2f 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -13,21 +13,8 @@ class GroqChat(OpenAICompatible): """Wrapper for Groq-hosted LLM models. Expects GROQ_API_KEY environment variable. - + See https://console.groq.com/docs/quickstart for more info on how to set up a Groq API key Uses the [OpenAI-compatible API](https://console.groq.com/docs/openai) - - To get started with this generator: - #. Visit [https://build.nvidia.com/explore/reasoning](build.nvidia.com/explore/reasoning) - and find the LLM you'd like to use. - #. On the page for the LLM you want to use (e.g. [mixtral-8x7b-instruct](https://build.nvidia.com/mistralai/mixtral-8x7b-instruct)), - click "Get API key" key above the code snippet. You may need to create an - account. Copy this key. - #. In your console, Set the ``NIM_API_KEY`` variable to this API key. On - Linux, this might look like ``export NIM_API_KEY="nvapi-xXxXxXx"``. - #. Run garak, setting ``--model_name`` to ``nim`` and ``--model_type`` to - the name of the model on [build.nvidia.com](https://build.nvidia.com/) - - e.g. ``mistralai/mixtral-8x7b-instruct-v0.1``. - """ # per https://console.groq.com/docs/openai From 5854159dc45105240a9af01f8435d24881626956 Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Wed, 4 Sep 2024 05:49:13 -0700 Subject: [PATCH 03/14] add tests and doc --- docs/source/garak.generators.groq.rst | 8 +++++++ docs/source/generators.rst | 1 + tests/generators/test_groq.py | 31 +++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 docs/source/garak.generators.groq.rst create mode 100644 tests/generators/test_groq.py diff --git a/docs/source/garak.generators.groq.rst b/docs/source/garak.generators.groq.rst new file mode 100644 index 00000000..8349b78c --- /dev/null +++ b/docs/source/garak.generators.groq.rst @@ -0,0 +1,8 @@ +garak.generators.nim +==================== + +.. automodule:: garak.generators.nim + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/source/generators.rst b/docs/source/generators.rst index 1cdd72e6..c5545187 100644 --- a/docs/source/generators.rst +++ b/docs/source/generators.rst @@ -13,6 +13,7 @@ For a detailed oversight into how a generator operates, see :ref:`garak.generato garak.generators.cohere garak.generators.function garak.generators.ggml + garak.generators.groq garak.generators.guardrails garak.generators.huggingface garak.generators.langchain diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py new file mode 100644 index 00000000..9cb97af0 --- /dev/null +++ b/tests/generators/test_groq.py @@ -0,0 +1,31 @@ +import os +import pytest + +import garak.cli +from garak.generators.groq import GroqChat + + +@pytest.mark.skipif( + os.getenv(GroqChat.ENV_VAR, None) is None, + reason=f"GroqChat API key is not set in {GroqChat.ENV_VAR}", +) +def test_groq_instantiate(): + g = GroqChat(name="llama3-8b-8192") + + +@pytest.mark.skipif( + os.getenv(GroqChat.ENV_VAR, None) is None, + reason=f"GroqChat API key is not set in {GroqChat.ENV_VAR}", +) +def test_groq_generate_1(): + g = GroqChat(name="llama3-8b-8192") + result = g._call_model("this is a test", generations_this_call=1) + assert isinstance(result, list), "GroqChat _call_model should return a list" + assert len(result) == 1, "GroqChat _call_model result list should have one item" + assert isinstance(result[0], str), "GroqChat _call_model should return a list" + result = g.generate("this is a test", generations_this_call=1) + assert isinstance(result, list), "GroqChat generate() should return a list" + assert ( + len(result) == 1 + ), "GroqChat generate() result list should have one item when generations_this_call=1" + assert isinstance(result[0], str), "GroqChat generate() should return a list" From 7489c09f85d6a7869ad8a1ce7fe22cb7392ad247 Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Wed, 4 Sep 2024 06:29:37 -0700 Subject: [PATCH 04/14] update comments --- garak/generators/groq.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index 2b9c4a2f..9a20028e 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -1,7 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NVIDIA Inference Microservice LLM interface""" +"""GroqChat API support""" import random from typing import List, Union From b6a26768ce97e9afc6359bccc3c75b9e8d270bfa Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Wed, 4 Sep 2024 06:46:26 -0700 Subject: [PATCH 05/14] fix generator docs --- docs/source/garak.generators.groq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/garak.generators.groq.rst b/docs/source/garak.generators.groq.rst index 8349b78c..1dba3acb 100644 --- a/docs/source/garak.generators.groq.rst +++ b/docs/source/garak.generators.groq.rst @@ -1,7 +1,7 @@ -garak.generators.nim +garak.generators.groq ==================== -.. automodule:: garak.generators.nim +.. automodule:: garak.generators.groq :members: :undoc-members: :show-inheritance: From d2f574b83cfc5761ff3b2e7a6ef5badb3c706c8a Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:17:20 +0200 Subject: [PATCH 06/14] add copyright info for groq generator Co-authored-by: Jeffrey Martin Signed-off-by: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> --- garak/generators/groq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index 9a20028e..cec9b670 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """GroqChat API support""" import random From 8d2d519a1e0e75f8de7528837234b0f6fe882d5c Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:17:55 +0200 Subject: [PATCH 07/14] add copyright info for Groq generator tests Co-authored-by: Jeffrey Martin Signed-off-by: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> --- tests/generators/test_groq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py index 9cb97af0..2d7b8efc 100644 --- a/tests/generators/test_groq.py +++ b/tests/generators/test_groq.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import os import pytest From 8defde123f12df16c555a153208f6eb59f229bd7 Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:20:10 +0200 Subject: [PATCH 08/14] reformat docs Co-authored-by: Jeffrey Martin Signed-off-by: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> --- garak/generators/groq.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index cec9b670..9c339178 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -12,7 +12,9 @@ class GroqChat(OpenAICompatible): - """Wrapper for Groq-hosted LLM models. Expects GROQ_API_KEY environment variable. + """Wrapper for Groq-hosted LLM models. + + Expects GROQ_API_KEY environment variable. See https://console.groq.com/docs/quickstart for more info on how to set up a Groq API key Uses the [OpenAI-compatible API](https://console.groq.com/docs/openai) """ From 3c0b01d819cf1e95a5aba1954d0e584dadb00ede Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:22:32 +0200 Subject: [PATCH 09/14] more descriptive test info Co-authored-by: Jeffrey Martin Signed-off-by: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> --- tests/generators/test_groq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py index 2d7b8efc..a880d7b3 100644 --- a/tests/generators/test_groq.py +++ b/tests/generators/test_groq.py @@ -25,7 +25,7 @@ def test_groq_generate_1(): result = g._call_model("this is a test", generations_this_call=1) assert isinstance(result, list), "GroqChat _call_model should return a list" assert len(result) == 1, "GroqChat _call_model result list should have one item" - assert isinstance(result[0], str), "GroqChat _call_model should return a list" + assert isinstance(result[0], str), "GroqChat generate() should contain a str" result = g.generate("this is a test", generations_this_call=1) assert isinstance(result, list), "GroqChat generate() should return a list" assert ( From df56c0678e374f1de6352fe0b7164c286daf47fb Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:22:50 +0200 Subject: [PATCH 10/14] more descriptive test info Co-authored-by: Jeffrey Martin Signed-off-by: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> --- tests/generators/test_groq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py index a880d7b3..1edf8d40 100644 --- a/tests/generators/test_groq.py +++ b/tests/generators/test_groq.py @@ -31,4 +31,4 @@ def test_groq_generate_1(): assert ( len(result) == 1 ), "GroqChat generate() result list should have one item when generations_this_call=1" - assert isinstance(result[0], str), "GroqChat generate() should return a list" + assert isinstance(result[0], str), "GroqChat generate() should contain a str" From f787670a45681cac1a9cf20b7e56fc858c8b6b3e Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Thu, 5 Sep 2024 02:15:03 -0700 Subject: [PATCH 11/14] add groqchat to openai compatible tests and test n>1 completions --- tests/generators/test_groq.py | 10 ++++++++++ tests/generators/test_openai_compatible.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py index 1edf8d40..a9982d4d 100644 --- a/tests/generators/test_groq.py +++ b/tests/generators/test_groq.py @@ -3,11 +3,21 @@ import os import pytest +import httpx import garak.cli from garak.generators.groq import GroqChat +def test_groq_invalid_multiple_completions(): + with pytest.raises(AssertionError) as e_info: + generator = GroqChat(name="llama3-8b-8192") + generator._call_model( + prompt="this is expected to fail", generations_this_call=2 + ) + assert "n > 1 is not supported" in str(e_info.value) + + @pytest.mark.skipif( os.getenv(GroqChat.ENV_VAR, None) is None, reason=f"GroqChat API key is not set in {GroqChat.ENV_VAR}", diff --git a/tests/generators/test_openai_compatible.py b/tests/generators/test_openai_compatible.py index 1da257a1..db676da5 100644 --- a/tests/generators/test_openai_compatible.py +++ b/tests/generators/test_openai_compatible.py @@ -16,7 +16,7 @@ # GENERATORS = [ # classname for (classname, active) in _plugins.enumerate_plugins("generators") # ] -GENERATORS = ["generators.openai.OpenAIGenerator", "generators.nim.NVOpenAIChat"] +GENERATORS = ["generators.openai.OpenAIGenerator", "generators.nim.NVOpenAIChat", "generators.groq.GroqChat"] MODEL_NAME = "gpt-3.5-turbo-instruct" ENV_VAR = os.path.abspath( From b1bb30771c31cf276bbaf0fdafc1b6db1b372955 Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:16:26 +0200 Subject: [PATCH 12/14] remove unused timeout Co-authored-by: Jeffrey Martin Signed-off-by: Mihailo Milenkovic <167300017+mmilenkovic-groq@users.noreply.github.com> --- garak/generators/groq.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index 9c339178..90081c1c 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -41,8 +41,6 @@ class GroqChat(OpenAICompatible): supports_multiple_generations = False generator_family_name = "Groq" - timeout = 60 - def _load_client(self): self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key) if self.name in ("", None): From 7de7338e55104bc69bbd9b0a4f64662770ad1991 Mon Sep 17 00:00:00 2001 From: Mihailo Milenkovic Date: Thu, 5 Sep 2024 10:07:29 -0700 Subject: [PATCH 13/14] add missing env var in test --- tests/generators/test_groq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py index a9982d4d..1295f11d 100644 --- a/tests/generators/test_groq.py +++ b/tests/generators/test_groq.py @@ -10,6 +10,8 @@ def test_groq_invalid_multiple_completions(): + if os.getenv(GroqChat.ENV_VAR, None) is None: + os.environ[GroqChat.ENV_VAR] = "fake_api_key" with pytest.raises(AssertionError) as e_info: generator = GroqChat(name="llama3-8b-8192") generator._call_model( From 2a146835084eb14e771ec1c2d373441fe385b6a3 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Fri, 6 Sep 2024 15:19:56 +0200 Subject: [PATCH 14/14] rm nv spdx, spurious import --- garak/generators/groq.py | 5 +---- tests/generators/test_groq.py | 5 ----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index 90081c1c..28635965 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -1,6 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - """GroqChat API support""" import random @@ -13,7 +10,7 @@ class GroqChat(OpenAICompatible): """Wrapper for Groq-hosted LLM models. - + Expects GROQ_API_KEY environment variable. See https://console.groq.com/docs/quickstart for more info on how to set up a Groq API key Uses the [OpenAI-compatible API](https://console.groq.com/docs/openai) diff --git a/tests/generators/test_groq.py b/tests/generators/test_groq.py index 1295f11d..e8ac5d55 100644 --- a/tests/generators/test_groq.py +++ b/tests/generators/test_groq.py @@ -1,11 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - import os import pytest -import httpx -import garak.cli from garak.generators.groq import GroqChat