diff --git a/FirebaseAI/Tests/TestApp/Sources/Constants.swift b/FirebaseAI/Tests/TestApp/Sources/Constants.swift index 4f88a39448b..6ed79da5aae 100644 --- a/FirebaseAI/Tests/TestApp/Sources/Constants.swift +++ b/FirebaseAI/Tests/TestApp/Sources/Constants.swift @@ -30,5 +30,6 @@ public enum ModelNames { public static let gemini2_5_FlashLite = "gemini-2.5-flash-lite" public static let gemini2_5_FlashLivePreview = "gemini-live-2.5-flash-preview" public static let gemini2_5_Pro = "gemini-2.5-pro" + public static let gemini3FlashPreview = "gemini-3-flash-preview" public static let gemma3_4B = "gemma-3-4b-it" } diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index 046d49ff4c0..f8102f70229 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -53,10 +53,12 @@ struct GenerateContentIntegrationTests { (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), + (InstanceConfig.googleAI_v1beta, ModelNames.gemini3FlashPreview), + (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini3FlashPreview), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), // Note: The following configs are commented out for easy one-off manual testing. + // (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), // (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), @@ -82,10 +84,17 @@ struct GenerateContentIntegrationTests { let promptTokensDetails = try #require(usageMetadata.promptTokensDetails.first) #expect(promptTokensDetails.modality == .text) #expect(promptTokensDetails.tokenCount == usageMetadata.promptTokenCount) - #expect(usageMetadata.thoughtsTokenCount == 0) + if modelName.hasPrefix("gemini-3") { + #expect(usageMetadata.thoughtsTokenCount == 64) + } else { + #expect(usageMetadata.thoughtsTokenCount == 0) + } // The fields `candidatesTokenCount` and `candidatesTokensDetails` are not included when using // Gemma models. - if modelName.hasPrefix("gemma") { + if modelName.hasPrefix("gemini-3") { + #expect(usageMetadata.candidatesTokenCount == 2) + #expect(usageMetadata.candidatesTokensDetails.isEmpty) + } else if modelName.hasPrefix("gemma") { #expect(usageMetadata.candidatesTokenCount == 0) #expect(usageMetadata.candidatesTokensDetails.isEmpty) } else { @@ -95,9 +104,9 @@ struct GenerateContentIntegrationTests { #expect(candidatesTokensDetails.modality == .text) #expect(candidatesTokensDetails.tokenCount == usageMetadata.candidatesTokenCount) } - #expect(usageMetadata.totalTokenCount > 0) - #expect(usageMetadata.totalTokenCount == - (usageMetadata.promptTokenCount + usageMetadata.candidatesTokenCount)) + #expect(usageMetadata.totalTokenCount == (usageMetadata.promptTokenCount + + usageMetadata.candidatesTokenCount + + usageMetadata.thoughtsTokenCount)) } @Test( @@ -161,16 +170,21 @@ struct GenerateContentIntegrationTests { (.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig( thinkingBudget: 32768, includeThoughts: true )), - (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 0)), - ( - .googleAI_v1beta_freeTier, - ModelNames.gemini2_5_Flash, - ThinkingConfig(thinkingBudget: 24576) - ), - (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig( - thinkingBudget: 24576, includeThoughts: true + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig(thinkingBudget: 128)), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig(thinkingBudget: 32768)), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig( + thinkingBudget: 32768, includeThoughts: true )), // Note: The following configs are commented out for easy one-off manual testing. +// (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 0)), +// ( +// .googleAI_v1beta_freeTier, +// ModelNames.gemini2_5_Flash, +// ThinkingConfig(thinkingBudget: 24576) +// ), +// (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig( +// thinkingBudget: 24576, includeThoughts: true +// )), // (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( // thinkingBudget: 0 // )), @@ -255,6 +269,10 @@ struct GenerateContentIntegrationTests { (.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig( thinkingBudget: -1, includeThoughts: true )), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig(thinkingBudget: -1)), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig( + thinkingBudget: -1, includeThoughts: true + )), ] as [(InstanceConfig, String, ThinkingConfig)] ) func generateContentThinkingFunctionCalling(_ config: InstanceConfig, modelName: String, @@ -470,19 +488,19 @@ struct GenerateContentIntegrationTests { @Test(arguments: [ (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashLite), - (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashLite), - (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), + (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini3FlashPreview), + (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini3FlashPreview), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), - (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), // Note: The following configs are commented out for easy one-off manual testing. // (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemma3_4B), +// (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), +// (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), ]) func generateContentStream(_ config: InstanceConfig, modelName: String) async throws { let expectedResponse = [ diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift index fecf8e80e7b..7c661f205d0 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift @@ -91,7 +91,7 @@ struct LiveSessionTests { ) } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendTextRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -113,7 +113,7 @@ struct LiveSessionTests { #expect(modelResponse == "yes") } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendTextRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( @@ -136,7 +136,7 @@ struct LiveSessionTests { #expect(modelResponse == "yes") } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendAudioRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( @@ -165,7 +165,7 @@ struct LiveSessionTests { #expect(modelResponse == "goodbye") } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendAudioRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -192,7 +192,10 @@ struct LiveSessionTests { #expect(modelResponse == "goodbye") } - @Test(arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive }) + @Test( + .disabled("Temporarily disabled"), + arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive } + ) // gemini-2.0-flash-live-001 is buggy and likes to respond to the audio or system instruction // (eg; it will say 'okay' or 'hello', instead of following the instructions) func sendVideoRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { @@ -235,7 +238,7 @@ struct LiveSessionTests { #expect(["kitten", "cat", "kitty"].contains(modelResponse)) } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func realtime_functionCalling(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -283,7 +286,7 @@ struct LiveSessionTests { #expect(modelResponse == "smith") } - @Test(arguments: arguments.filter { + @Test(.disabled("Temporarily disabled"), arguments: arguments.filter { // TODO: (b/450982184) Remove when Vertex AI adds support for Function IDs and Cancellation switch $0.0.apiConfig.service { case .googleAI: @@ -325,6 +328,7 @@ struct LiveSessionTests { } @Test( + .disabled("Temporarily disabled"), arguments: arguments.filter { !$0.0.useLimitedUseAppCheckTokens } ) // Getting a limited use token adds too much of an overhead; we can't interrupt the model in time @@ -360,7 +364,7 @@ struct LiveSessionTests { } } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func incremental_works(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, diff --git a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift index 6522abbad1e..476be89b878 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift @@ -83,8 +83,8 @@ struct InstanceConfig: Equatable, Encodable { vertexAI_v1beta_global_appCheckLimitedUse, googleAI_v1beta, googleAI_v1beta_appCheckLimitedUse, - googleAI_v1beta_freeTier, // Note: The following configs are commented out for easy one-off manual testing. + // googleAI_v1beta_freeTier, // vertexAI_v1beta_staging, // vertexAI_v1beta_staging_global_bypassProxy, // googleAI_v1beta_staging,