Skip to content

Commit a16b307

Browse files
[.Net] Add Goolge gemini (#2868)
* update * add vertex gemini test * remove DTO * add test for vertexGeminiAgent * update test name * update IGeminiClient interface * add test for streaming * add message connector * add gemini message extension * add tests * update * add gemnini sample * update examples * add test for iamge * fix test * add more tests * add streaming message test * add comment * remove unused json * implement google gemini client * update * fix comment
1 parent 7d057a9 commit a16b307

File tree

44 files changed

+2530
-109
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2530
-109
lines changed

dotnet/AutoGen.sln

+20
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Anthropic.Tests", "
5353
EndProject
5454
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Anthropic.Samples", "sample\AutoGen.Anthropic.Samples\AutoGen.Anthropic.Samples.csproj", "{834B4E85-64E5-4382-8465-548F332E5298}"
5555
EndProject
56+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Gemini", "src\AutoGen.Gemini\AutoGen.Gemini.csproj", "{EFE0DC86-80FC-4D52-95B7-07654BA1A769}"
57+
EndProject
58+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Gemini.Tests", "test\AutoGen.Gemini.Tests\AutoGen.Gemini.Tests.csproj", "{8EA16BAB-465A-4C07-ABC4-1070D40067E9}"
59+
EndProject
60+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.Gemini.Sample", "sample\AutoGen.Gemini.Sample\AutoGen.Gemini.Sample.csproj", "{19679B75-CE3A-4DF0-A3F0-CA369D2760A4}"
5661
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.AotCompatibility.Tests", "test\AutoGen.AotCompatibility.Tests\AutoGen.AotCompatibility.Tests.csproj", "{6B82F26D-5040-4453-B21B-C8D1F913CE4C}"
5762
EndProject
5863
Global
@@ -149,6 +154,18 @@ Global
149154
{834B4E85-64E5-4382-8465-548F332E5298}.Debug|Any CPU.Build.0 = Debug|Any CPU
150155
{834B4E85-64E5-4382-8465-548F332E5298}.Release|Any CPU.ActiveCfg = Release|Any CPU
151156
{834B4E85-64E5-4382-8465-548F332E5298}.Release|Any CPU.Build.0 = Release|Any CPU
157+
{EFE0DC86-80FC-4D52-95B7-07654BA1A769}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
158+
{EFE0DC86-80FC-4D52-95B7-07654BA1A769}.Debug|Any CPU.Build.0 = Debug|Any CPU
159+
{EFE0DC86-80FC-4D52-95B7-07654BA1A769}.Release|Any CPU.ActiveCfg = Release|Any CPU
160+
{EFE0DC86-80FC-4D52-95B7-07654BA1A769}.Release|Any CPU.Build.0 = Release|Any CPU
161+
{8EA16BAB-465A-4C07-ABC4-1070D40067E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
162+
{8EA16BAB-465A-4C07-ABC4-1070D40067E9}.Debug|Any CPU.Build.0 = Debug|Any CPU
163+
{8EA16BAB-465A-4C07-ABC4-1070D40067E9}.Release|Any CPU.ActiveCfg = Release|Any CPU
164+
{8EA16BAB-465A-4C07-ABC4-1070D40067E9}.Release|Any CPU.Build.0 = Release|Any CPU
165+
{19679B75-CE3A-4DF0-A3F0-CA369D2760A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
166+
{19679B75-CE3A-4DF0-A3F0-CA369D2760A4}.Debug|Any CPU.Build.0 = Debug|Any CPU
167+
{19679B75-CE3A-4DF0-A3F0-CA369D2760A4}.Release|Any CPU.ActiveCfg = Release|Any CPU
168+
{19679B75-CE3A-4DF0-A3F0-CA369D2760A4}.Release|Any CPU.Build.0 = Release|Any CPU
152169
{6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
153170
{6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU
154171
{6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -180,6 +197,9 @@ Global
180197
{6A95E113-B824-4524-8F13-CD0C3E1C8804} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
181198
{815E937E-86D6-4476-9EC6-B7FBCBBB5DB6} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
182199
{834B4E85-64E5-4382-8465-548F332E5298} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9}
200+
{EFE0DC86-80FC-4D52-95B7-07654BA1A769} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
201+
{8EA16BAB-465A-4C07-ABC4-1070D40067E9} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
202+
{19679B75-CE3A-4DF0-A3F0-CA369D2760A4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9}
183203
{6B82F26D-5040-4453-B21B-C8D1F913CE4C} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
184204
EndGlobalSection
185205
GlobalSection(ExtensibilityGlobals) = postSolution

dotnet/Directory.Build.props

+25
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,37 @@
1313
<CSNoWarn>CS1998;CS1591</CSNoWarn>
1414
<NoWarn>$(NoWarn);$(CSNoWarn);NU5104</NoWarn>
1515
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
16+
<GenerateDocumentationFile>true</GenerateDocumentationFile>
1617
<IsPackable>false</IsPackable>
1718
<EnableNetAnalyzers>true</EnableNetAnalyzers>
1819
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
20+
<IsTestProject>false</IsTestProject>
1921
</PropertyGroup>
2022

2123
<PropertyGroup>
2224
<RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
2325
</PropertyGroup>
26+
27+
<ItemGroup Condition="'$(IsTestProject)' == 'true'">
28+
<PackageReference Include="ApprovalTests" Version="$(ApprovalTestVersion)" />
29+
<PackageReference Include="FluentAssertions" Version="$(FluentAssertionVersion)" />
30+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
31+
<PackageReference Include="xunit" Version="$(XUnitVersion)" />
32+
<PackageReference Include="xunit.runner.console" Version="$(XUnitVersion)" />
33+
<PackageReference Include="xunit.runner.visualstudio" Version="$(XUnitVersion)" />
34+
</ItemGroup>
35+
36+
<ItemGroup Condition="'$(IsTestProject)' == 'true'">
37+
<Content Include="$(RepoRoot)resource/**/*.*">
38+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
39+
<Link>testData/%(RecursiveDir)%(Filename)%(Extension)</Link>
40+
</Content>
41+
</ItemGroup>
42+
43+
<ItemGroup Condition="'$(IncludeResourceFolder)' == 'true'">
44+
<Content Include="$(RepoRoot)resource/**/*.*">
45+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
46+
<Link>resource/%(RecursiveDir)%(Filename)%(Extension)</Link>
47+
</Content>
48+
</ItemGroup>
2449
</Project>

dotnet/eng/Version.props

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<MicrosoftNETTestSdkVersion>17.7.0</MicrosoftNETTestSdkVersion>
1313
<MicrosoftDotnetInteractive>1.0.0-beta.24229.4</MicrosoftDotnetInteractive>
1414
<MicrosoftSourceLinkGitHubVersion>8.0.0</MicrosoftSourceLinkGitHubVersion>
15+
<GoogleCloudAPIPlatformVersion>3.0.0</GoogleCloudAPIPlatformVersion>
1516
<JsonSchemaVersion>4.3.0.2</JsonSchemaVersion>
1617
</PropertyGroup>
1718
</Project>

dotnet/resource/images/square.png

+3
Loading

dotnet/sample/AutoGen.BasicSamples/Example05_Dalle_And_GPT4V.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public static async Task RunAsync()
9393
if (reply.GetContent() is string content && content.Contains("IMAGE_GENERATION"))
9494
{
9595
var imageUrl = content.Split("\n").Last();
96-
var imageMessage = new ImageMessage(Role.Assistant, imageUrl, from: reply.From);
96+
var imageMessage = new ImageMessage(Role.Assistant, imageUrl, from: reply.From, mimeType: "image/png");
9797

9898
Console.WriteLine($"download image from {imageUrl} to {imagePath}");
9999
var httpClient = new HttpClient();
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<IncludeResourceFolder>true</IncludeResourceFolder>
9+
<GenerateDocumentationFile>True</GenerateDocumentationFile>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<ProjectReference Include="..\..\src\AutoGen\AutoGen.csproj" />
14+
<ProjectReference Include="..\..\src\AutoGen.Gemini\AutoGen.Gemini.csproj" />
15+
<ProjectReference Include="..\..\src\AutoGen.SourceGenerator\AutoGen.SourceGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
16+
<PackageReference Include="FluentAssertions" Version="$(FluentAssertionVersion)" />
17+
</ItemGroup>
18+
19+
</Project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Chat_With_Google_Gemini.cs
3+
4+
using AutoGen.Core;
5+
using AutoGen.Gemini.Middleware;
6+
using FluentAssertions;
7+
8+
namespace AutoGen.Gemini.Sample;
9+
10+
public class Chat_With_Google_Gemini
11+
{
12+
public static async Task RunAsync()
13+
{
14+
var apiKey = Environment.GetEnvironmentVariable("GOOGLE_GEMINI_API_KEY");
15+
16+
if (apiKey is null)
17+
{
18+
Console.WriteLine("Please set GOOGLE_GEMINI_API_KEY environment variable.");
19+
return;
20+
}
21+
22+
#region Create_Gemini_Agent
23+
var geminiAgent = new GeminiChatAgent(
24+
name: "gemini",
25+
model: "gemini-1.5-flash-001",
26+
apiKey: apiKey,
27+
systemMessage: "You are a helpful C# engineer, put your code between ```csharp and ```, don't explain the code")
28+
.RegisterMessageConnector()
29+
.RegisterPrintMessage();
30+
#endregion Create_Gemini_Agent
31+
32+
var reply = await geminiAgent.SendAsync("Can you write a piece of C# code to calculate 100th of fibonacci?");
33+
34+
#region verify_reply
35+
reply.Should().BeOfType<TextMessage>();
36+
#endregion verify_reply
37+
}
38+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Chat_With_Vertex_Gemini.cs
3+
4+
using AutoGen.Core;
5+
using AutoGen.Gemini.Middleware;
6+
using FluentAssertions;
7+
8+
namespace AutoGen.Gemini.Sample;
9+
10+
public class Chat_With_Vertex_Gemini
11+
{
12+
public static async Task RunAsync()
13+
{
14+
var projectID = Environment.GetEnvironmentVariable("GCP_VERTEX_PROJECT_ID");
15+
16+
if (projectID is null)
17+
{
18+
Console.WriteLine("Please set GCP_VERTEX_PROJECT_ID environment variable.");
19+
return;
20+
}
21+
22+
#region Create_Gemini_Agent
23+
var geminiAgent = new GeminiChatAgent(
24+
name: "gemini",
25+
model: "gemini-1.5-flash-001",
26+
location: "us-east1",
27+
project: projectID,
28+
systemMessage: "You are a helpful C# engineer, put your code between ```csharp and ```, don't explain the code")
29+
.RegisterMessageConnector()
30+
.RegisterPrintMessage();
31+
#endregion Create_Gemini_Agent
32+
33+
var reply = await geminiAgent.SendAsync("Can you write a piece of C# code to calculate 100th of fibonacci?");
34+
35+
#region verify_reply
36+
reply.Should().BeOfType<TextMessage>();
37+
#endregion verify_reply
38+
}
39+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Function_Call_With_Gemini.cs
3+
4+
using AutoGen.Core;
5+
using AutoGen.Gemini.Middleware;
6+
using FluentAssertions;
7+
using Google.Cloud.AIPlatform.V1;
8+
9+
namespace AutoGen.Gemini.Sample;
10+
11+
public partial class MovieFunction
12+
{
13+
/// <summary>
14+
/// find movie titles currently playing in theaters based on any description, genre, title words, etc.
15+
/// </summary>
16+
/// <param name="location">The city and state, e.g. San Francisco, CA or a zip code e.g. 95616</param>
17+
/// <param name="description">Any kind of description including category or genre, title words, attributes, etc.</param>
18+
/// <returns></returns>
19+
[Function]
20+
public async Task<string> FindMovies(string location, string description)
21+
{
22+
// dummy implementation
23+
var movies = new List<string> { "Barbie", "Spiderman", "Batman" };
24+
var result = $"Movies playing in {location} based on {description} are: {string.Join(", ", movies)}";
25+
26+
return result;
27+
}
28+
29+
/// <summary>
30+
/// find theaters based on location and optionally movie title which is currently playing in theaters
31+
/// </summary>
32+
/// <param name="location">The city and state, e.g. San Francisco, CA or a zip code e.g. 95616</param>
33+
/// <param name="movie">Any movie title</param>
34+
[Function]
35+
public async Task<string> FindTheaters(string location, string movie)
36+
{
37+
// dummy implementation
38+
var theaters = new List<string> { "AMC", "Regal", "Cinemark" };
39+
var result = $"Theaters playing {movie} in {location} are: {string.Join(", ", theaters)}";
40+
41+
return result;
42+
}
43+
44+
/// <summary>
45+
/// Find the start times for movies playing in a specific theater
46+
/// </summary>
47+
/// <param name="location">The city and state, e.g. San Francisco, CA or a zip code e.g. 95616</param>
48+
/// <param name="movie">Any movie title</param>
49+
/// <param name="theater">Name of the theater</param>
50+
/// <param name="date">Date for requested showtime</param>
51+
/// <returns></returns>
52+
[Function]
53+
public async Task<string> GetShowtimes(string location, string movie, string theater, string date)
54+
{
55+
// dummy implementation
56+
var showtimes = new List<string> { "10:00 AM", "12:00 PM", "2:00 PM", "4:00 PM", "6:00 PM", "8:00 PM" };
57+
var result = $"Showtimes for {movie} at {theater} in {location} are: {string.Join(", ", showtimes)}";
58+
59+
return result;
60+
}
61+
62+
}
63+
64+
/// <summary>
65+
/// Modified from https://ai.google.dev/gemini-api/docs/function-calling
66+
/// </summary>
67+
public partial class Function_Call_With_Gemini
68+
{
69+
public static async Task RunAsync()
70+
{
71+
var projectID = Environment.GetEnvironmentVariable("GCP_VERTEX_PROJECT_ID");
72+
73+
if (projectID is null)
74+
{
75+
Console.WriteLine("Please set GCP_VERTEX_PROJECT_ID environment variable.");
76+
return;
77+
}
78+
79+
var movieFunction = new MovieFunction();
80+
var functionMiddleware = new FunctionCallMiddleware(
81+
functions: [
82+
movieFunction.FindMoviesFunctionContract,
83+
movieFunction.FindTheatersFunctionContract,
84+
movieFunction.GetShowtimesFunctionContract
85+
],
86+
functionMap: new Dictionary<string, Func<string, Task<string>>>
87+
{
88+
{ movieFunction.FindMoviesFunctionContract.Name!, movieFunction.FindMoviesWrapper },
89+
{ movieFunction.FindTheatersFunctionContract.Name!, movieFunction.FindTheatersWrapper },
90+
{ movieFunction.GetShowtimesFunctionContract.Name!, movieFunction.GetShowtimesWrapper },
91+
});
92+
93+
#region Create_Gemini_Agent
94+
var geminiAgent = new GeminiChatAgent(
95+
name: "gemini",
96+
model: "gemini-1.5-flash-001",
97+
location: "us-central1",
98+
project: projectID,
99+
systemMessage: "You are a helpful AI assistant",
100+
toolConfig: new ToolConfig()
101+
{
102+
FunctionCallingConfig = new FunctionCallingConfig()
103+
{
104+
Mode = FunctionCallingConfig.Types.Mode.Auto,
105+
}
106+
})
107+
.RegisterMessageConnector()
108+
.RegisterPrintMessage()
109+
.RegisterStreamingMiddleware(functionMiddleware);
110+
#endregion Create_Gemini_Agent
111+
112+
#region Single_turn
113+
var question = new TextMessage(Role.User, "What movies are showing in North Seattle tonight?");
114+
var functionCallReply = await geminiAgent.SendAsync(question);
115+
#endregion Single_turn
116+
117+
#region Single_turn_verify_reply
118+
functionCallReply.Should().BeOfType<ToolCallAggregateMessage>();
119+
#endregion Single_turn_verify_reply
120+
121+
#region Multi_turn
122+
var finalReply = await geminiAgent.SendAsync(chatHistory: [question, functionCallReply]);
123+
#endregion Multi_turn
124+
125+
#region Multi_turn_verify_reply
126+
finalReply.Should().BeOfType<TextMessage>();
127+
#endregion Multi_turn_verify_reply
128+
}
129+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Image_Chat_With_Vertex_Gemini.cs
3+
4+
using AutoGen.Core;
5+
using AutoGen.Gemini.Middleware;
6+
using FluentAssertions;
7+
8+
namespace AutoGen.Gemini.Sample;
9+
10+
public class Image_Chat_With_Vertex_Gemini
11+
{
12+
public static async Task RunAsync()
13+
{
14+
var projectID = Environment.GetEnvironmentVariable("GCP_VERTEX_PROJECT_ID");
15+
16+
if (projectID is null)
17+
{
18+
Console.WriteLine("Please set GCP_VERTEX_PROJECT_ID environment variable.");
19+
return;
20+
}
21+
22+
#region Create_Gemini_Agent
23+
var geminiAgent = new GeminiChatAgent(
24+
name: "gemini",
25+
model: "gemini-1.5-flash-001",
26+
location: "us-east4",
27+
project: projectID,
28+
systemMessage: "You explain image content to user")
29+
.RegisterMessageConnector()
30+
.RegisterPrintMessage();
31+
#endregion Create_Gemini_Agent
32+
33+
#region Send_Image_Request
34+
var imagePath = Path.Combine("resource", "images", "background.png");
35+
var image = await File.ReadAllBytesAsync(imagePath);
36+
var imageMessage = new ImageMessage(Role.User, BinaryData.FromBytes(image, "image/png"));
37+
var reply = await geminiAgent.SendAsync("what's in the image", [imageMessage]);
38+
#endregion Send_Image_Request
39+
40+
#region Verify_Reply
41+
reply.Should().BeOfType<TextMessage>();
42+
#endregion Verify_Reply
43+
}
44+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Program.cs
3+
4+
using AutoGen.Gemini.Sample;
5+
6+
Image_Chat_With_Vertex_Gemini.RunAsync().Wait();

0 commit comments

Comments
 (0)