From 0651a4c344adec7cb17be473b2c66e9d6c04fb11 Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Tue, 30 Jul 2024 18:35:22 +0000 Subject: [PATCH 1/6] add openai-gemini example --- .../cloud-gemini_vertexai.ipynb | 125 +++++++++++++++++- 1 file changed, 122 insertions(+), 3 deletions(-) diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index e618966dc6cc..56a12c48140d 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -150,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -166,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -188,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -395,6 +395,125 @@ { "cell_type": "markdown", "metadata": {}, + "source": [ + "# Use Gemini via the OpenAI Library in Autogen\n", + "Using Gemini via the OpenAI library is also possible once you are already authenticated.
\n", + "The prerequisites are essentially the same as in the example above.
\n", + "\n", + "Run `gcloud auth application-default login` to set up application default credentials locally for the example below.
\n", + "\n", + "You can read more on the topic in the [official Google docs](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-gemini-using-openai-library).\n", + "
A list of currently supported models can also be found in the [docs](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-gemini-using-openai-library#supported_models)\n", + "\n", + "

\n", + "Note, that you will need to refresh your token regularly, by default every 1 hour." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "import google.auth\n", + "\n", + "scopes = [\"https://www.googleapis.com/auth/cloud-platform\"]\n", + "creds, project = google.auth.default(scopes)\n", + "auth_req = google.auth.transport.requests.Request()\n", + "creds.refresh(auth_req)\n", + "location = \"us-west1\"\n", + "prompt_price_per_1k = (\n", + " 0.000125 # For more up-to-date prices see https://cloud.google.com/vertex-ai/generative-ai/pricing\n", + ")\n", + "completion_token_price_per_1k = (\n", + " 0.000375 # For more up-to-date prices see https://cloud.google.com/vertex-ai/generative-ai/pricing\n", + ")\n", + "\n", + "openai_gemini_config = [\n", + " {\n", + " \"model\": \"google/gemini-1.5-pro-001\",\n", + " \"api_type\": \"openai\",\n", + " \"base_url\": f\"https://{location}-aiplatform.googleapis.com/v1beta1/projects/{project}/locations/{location}/endpoints/openapi\",\n", + " \"api_key\": creds.token,\n", + " \"price\": [prompt_price_per_1k, completion_token_price_per_1k],\n", + " }\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "\n", + " Compute the integral of the function f(x)=x^3 on the interval 0 to 10 using a Python script,\n", + " which returns the value of the definite integral.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + "```python\n", + "# filename: integral.py\n", + "def integrate_x_cubed(a, b):\n", + " \"\"\"\n", + " This function calculates the definite integral of x^3 from a to b.\n", + "\n", + " Args:\n", + " a: The lower limit of integration.\n", + " b: The upper limit of integration.\n", + "\n", + " Returns:\n", + " The value of the definite integral.\n", + " \"\"\"\n", + " return (b**4 - a**4) / 4\n", + "\n", + "# Calculate the integral of x^3 from 0 to 10\n", + "result = integrate_x_cubed(0, 10)\n", + "\n", + "# Print the result\n", + "print(result)\n", + "```\n", + "\n", + "This script defines a function `integrate_x_cubed` that takes the lower and upper limits of integration as arguments and returns the definite integral of x^3 using the power rule of integration. The script then calls this function with the limits 0 and 10 and prints the result.\n", + "\n", + "Execute the script `python integral.py`, you should get the result: `2500.0`.\n", + "\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "assistant = AssistantAgent(\"assistant\", llm_config={\"config_list\": openai_gemini_config}, max_consecutive_auto_reply=3)\n", + "\n", + "user_proxy = UserProxyAgent(\n", + " \"user_proxy\",\n", + " code_execution_config={\"work_dir\": \"coding\", \"use_docker\": False},\n", + " human_input_mode=\"NEVER\",\n", + " is_termination_msg=lambda x: content_str(x.get(\"content\")).find(\"TERMINATE\") >= 0,\n", + ")\n", + "\n", + "result = user_proxy.initiate_chat(\n", + " assistant,\n", + " message=\"\"\"\n", + " Compute the integral of the function f(x)=x^3 on the interval 0 to 10 using a Python script,\n", + " which returns the value of the definite integral.\"\"\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [] } ], From 883157370f6471fb56e5d1f4ad38b9cb8625ede9 Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Fri, 2 Aug 2024 18:38:54 +0000 Subject: [PATCH 2/6] fix exec numbering --- .../cloud-gemini_vertexai.ipynb | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index 56a12c48140d..be43a88463ae 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -150,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -166,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -188,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -398,9 +398,8 @@ "source": [ "# Use Gemini via the OpenAI Library in Autogen\n", "Using Gemini via the OpenAI library is also possible once you are already authenticated.
\n", - "The prerequisites are essentially the same as in the example above.
\n", - "\n", "Run `gcloud auth application-default login` to set up application default credentials locally for the example below.
\n", + "The prerequisites are essentially the same as in the example above.
\n", "\n", "You can read more on the topic in the [official Google docs](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-gemini-using-openai-library).\n", "
A list of currently supported models can also be found in the [docs](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-gemini-using-openai-library#supported_models)\n", @@ -411,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -442,7 +441,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -455,7 +454,13 @@ " Compute the integral of the function f(x)=x^3 on the interval 0 to 10 using a Python script,\n", " which returns the value of the definite integral.\n", "\n", - "--------------------------------------------------------------------------------\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001b[33massistant\u001b[0m (to user_proxy):\n", "\n", "```python\n", @@ -508,13 +513,6 @@ " which returns the value of the definite integral.\"\"\",\n", ")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 607ac259a93553bb04aed0b6d2d0de400c0f6c24 Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Sat, 3 Aug 2024 00:05:12 +0000 Subject: [PATCH 3/6] improve isntructions --- .../cloud-gemini_vertexai.ipynb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index be43a88463ae..27f0b4ccdc7d 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -91,7 +91,7 @@ "\n", "#### Use the Google Default Credentials\n", "\n", - "If you are using [Cloud Shell](https://shell.cloud.google.com/cloudshell) or [Cloud Shell editor](https://shell.cloud.google.com/cloudshell/editor) in Google Cloud,
then you are already authenticated. If you have the Google Cloud SDK installed locally,
then you can login by running `gcloud auth login` in the command line. \n", + "If you are using [Cloud Shell](https://shell.cloud.google.com/cloudshell) or [Cloud Shell editor](https://shell.cloud.google.com/cloudshell/editor) in Google Cloud,
then you are already authenticated. If you have the Google Cloud SDK installed locally,
then you can login by running `gcloud auth application-default login` in the command line. \n", "\n", "Detailed instructions for installing the Google Cloud SDK can be found [here](https://cloud.google.com/sdk/docs/install).\n", "\n", @@ -99,7 +99,7 @@ "\n", "The google-auth library supports a wide range of authentication scenarios, and you can simply pass a previously created `Credentials` object to the `llm_config`.
\n", "The [official documentation](https://google-auth.readthedocs.io/) of the Python package provides a detailed overview of the supported methods and usage examples.
\n", - "If you are already authenticated, like in [Cloud Shell](https://shell.cloud.google.com/cloudshell), or after running the `gcloud auth login` command in a CLI, then the `google.auth.default()` Python method will automatically return your currently active credentials." + "If you are already authenticated, like in [Cloud Shell](https://shell.cloud.google.com/cloudshell), or after running the `gcloud auth application-default login` command in a CLI, then the `google.auth.default()` Python method will automatically return your currently active credentials." ] }, { @@ -311,10 +311,10 @@ "The `GOOGLE_APPLICATION_CREDENTIALS` environment variable is a path to our service account JSON keyfile, as described in the [Use Service Account Keyfile](#Use Service Account Keyfile) section above.
\n", "We also need to set the Google cloud project, which is `autogen-with-gemini` in this example.

\n", "\n", - "Note, we could also run `gcloud auth login` in case we wish to use our personal Google account instead of a service account.\n", + "Note, we could also run `gcloud auth application-default login` to use our personal Google account instead of a service account.\n", "In this case we need to run the following commands:\n", "```bash\n", - "gcloud auth login\n", + "gcloud gcloud auth application-default login\n", "gcloud config set project autogen-with-gemini\n", "```" ] @@ -399,12 +399,16 @@ "# Use Gemini via the OpenAI Library in Autogen\n", "Using Gemini via the OpenAI library is also possible once you are already authenticated.
\n", "Run `gcloud auth application-default login` to set up application default credentials locally for the example below.
\n", + "Also set the Google cloud project on the CLI if you have not done so far:
\n", + "```bash\n", + "gcloud config set project autogen-with-gemini\n", + "```\n", "The prerequisites are essentially the same as in the example above.
\n", "\n", "You can read more on the topic in the [official Google docs](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-gemini-using-openai-library).\n", "
A list of currently supported models can also be found in the [docs](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-gemini-using-openai-library#supported_models)\n", - "\n", - "

\n", + "
\n", + "
\n", "Note, that you will need to refresh your token regularly, by default every 1 hour." ] }, From 301dddbcb7ff7a6b98c32f1a3392d2b4af60a48a Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Mon, 5 Aug 2024 18:35:29 +0000 Subject: [PATCH 4/6] fix br tag --- .../docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index 27f0b4ccdc7d..59c7b58faae9 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -399,7 +399,7 @@ "# Use Gemini via the OpenAI Library in Autogen\n", "Using Gemini via the OpenAI library is also possible once you are already authenticated.
\n", "Run `gcloud auth application-default login` to set up application default credentials locally for the example below.
\n", - "Also set the Google cloud project on the CLI if you have not done so far:
\n", + "Also set the Google cloud project on the CLI if you have not done so far:
\n", "```bash\n", "gcloud config set project autogen-with-gemini\n", "```\n", From 47fde0ed47d989b96f8c3de3e663c1c8519220b5 Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Mon, 5 Aug 2024 22:43:12 +0000 Subject: [PATCH 5/6] mention roles/aiplatform.user and fix markdown reference --- .../non-openai-models/cloud-gemini_vertexai.ipynb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index 59c7b58faae9..765404c39ce5 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -62,7 +62,12 @@ "\n", "\n", "\n", - "For the sake of simplicity we will assign the Editor role to our service account for autogen on our Autogen-with-Gemini Google Cloud project.\n", + "For the sake of simplicity we can assign the [Editor role](https://cloud.google.com/iam/docs/understanding-roles#editor) to our service account for autogen on our Autogen-with-Gemini Google Cloud project.
\n", + "If more fine grained access control is required, one can also just assign the [roles/aiplatform.user](https://cloud.google.com/vertex-ai/docs/general/access-control#aiplatform.user) role to the service account used by autogen in our Google Cloud project instead:\n", + "```bash\n", + "gcloud projects add-iam-policy-binding autogen-with-gemini \\\n", + " --member=serviceAccount:autogen@autogen-with-gemini.iam.gserviceaccount.com --role roles/aiplatform.user\n", + "```\n", "\n", "* Under IAM & Admin > Service Account select the newly created service accounts, and click the option \"Manage keys\" among the items. \n", "* From the \"ADD KEY\" dropdown select \"Create new key\" and select the JSON format and click CREATE.\n", @@ -83,7 +88,7 @@ "Additionally, AutoGen also supports authentication using `Credentials` objects in Python with the [google-auth library](https://google-auth.readthedocs.io/), which enables even more flexibility.
\n", "For example, we can even use impersonated credentials.\n", "\n", - "#### Use Service Account Keyfile\n", + "#### Use Service Account Keyfile\n", "\n", "The Google Cloud service account can be specified by setting the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to the JSON key file of the service account.
\n", "\n", @@ -308,7 +313,7 @@ "gcloud auth application-default login\n", "gcloud config set project autogen-with-gemini\n", "```\n", - "The `GOOGLE_APPLICATION_CREDENTIALS` environment variable is a path to our service account JSON keyfile, as described in the [Use Service Account Keyfile](#Use Service Account Keyfile) section above.
\n", + "The `GOOGLE_APPLICATION_CREDENTIALS` environment variable is a path to our service account JSON keyfile, as described in the [Use Service Account Keyfile](#use_svc_keyfile) section above.
\n", "We also need to set the Google cloud project, which is `autogen-with-gemini` in this example.

\n", "\n", "Note, we could also run `gcloud auth application-default login` to use our personal Google account instead of a service account.\n", From acd1270f0889013592b01da014ebc1b8f1db3aa3 Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Sun, 11 Aug 2024 22:03:57 +0000 Subject: [PATCH 6/6] remove mentioning the editor role, and only use the Vertex AI User role --- .../docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index 765404c39ce5..2d32351784d9 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -62,8 +62,8 @@ "\n", "\n", "\n", - "For the sake of simplicity we can assign the [Editor role](https://cloud.google.com/iam/docs/understanding-roles#editor) to our service account for autogen on our Autogen-with-Gemini Google Cloud project.
\n", - "If more fine grained access control is required, one can also just assign the [roles/aiplatform.user](https://cloud.google.com/vertex-ai/docs/general/access-control#aiplatform.user) role to the service account used by autogen in our Google Cloud project instead:\n", + "Next we assign the [Vertex AI User](https://cloud.google.com/vertex-ai/docs/general/access-control#aiplatform.user) for the service account. This can be done in the [Google Cloud console](https://console.cloud.google.com/iam-admin/iam?project=autogen-with-gemini) in our `autogen-with-gemini` project.
\n", + "Alternatively, we can also grant the [Vertex AI User](https://cloud.google.com/vertex-ai/docs/general/access-control#aiplatform.user) role by running a command using the gcloud CLI, for example in [Cloud Shell](https://shell.cloud.google.com/cloudshell):\n", "```bash\n", "gcloud projects add-iam-policy-binding autogen-with-gemini \\\n", " --member=serviceAccount:autogen@autogen-with-gemini.iam.gserviceaccount.com --role roles/aiplatform.user\n",