From 47992c8e3f05704ab00831767fe30512765414ad Mon Sep 17 00:00:00 2001 From: Paul Beusterien Date: Mon, 22 Dec 2025 18:26:42 -0800 Subject: [PATCH 1/6] [AI] Make Integration tests green --- .../project.pbxproj | 10 ++++ .../Tests/TestApp/Sources/Constants.swift | 1 + .../GenerateContentIntegrationTests.swift | 46 ++++++++++++------- .../Tests/Utilities/InstanceConfig.swift | 2 +- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj b/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj index c40918762a4..8b59429f1e5 100644 --- a/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj +++ b/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj @@ -36,6 +36,13 @@ ); target = 866138682CC943DE00F4B78E /* IntegrationTests-SPM */; }; + DECF58E22EFA338E000A3EC4 /* Exceptions for "Tests" folder in "IntegrationTests-SPM" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Integration/LiveSessionTests.swift, + ); + target = 866138682CC943DE00F4B78E /* IntegrationTests-SPM */; + }; /* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -46,6 +53,9 @@ }; 863E93F42EC69AB100BE4F4E /* Tests */ = { isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + DECF58E22EFA338E000A3EC4 /* Exceptions for "Tests" folder in "IntegrationTests-SPM" target */, + ); path = Tests; sourceTree = ""; }; diff --git a/FirebaseAI/Tests/TestApp/Sources/Constants.swift b/FirebaseAI/Tests/TestApp/Sources/Constants.swift index 4f88a39448b..6ed79da5aae 100644 --- a/FirebaseAI/Tests/TestApp/Sources/Constants.swift +++ b/FirebaseAI/Tests/TestApp/Sources/Constants.swift @@ -30,5 +30,6 @@ public enum ModelNames { public static let gemini2_5_FlashLite = "gemini-2.5-flash-lite" public static let gemini2_5_FlashLivePreview = "gemini-live-2.5-flash-preview" public static let gemini2_5_Pro = "gemini-2.5-pro" + public static let gemini3FlashPreview = "gemini-3-flash-preview" public static let gemma3_4B = "gemma-3-4b-it" } diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index 046d49ff4c0..ae214a2ff2c 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -53,10 +53,12 @@ struct GenerateContentIntegrationTests { (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), + (InstanceConfig.googleAI_v1beta, ModelNames.gemini3FlashPreview), + (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini3FlashPreview), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), // Note: The following configs are commented out for easy one-off manual testing. + // (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), // (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), @@ -82,10 +84,17 @@ struct GenerateContentIntegrationTests { let promptTokensDetails = try #require(usageMetadata.promptTokensDetails.first) #expect(promptTokensDetails.modality == .text) #expect(promptTokensDetails.tokenCount == usageMetadata.promptTokenCount) - #expect(usageMetadata.thoughtsTokenCount == 0) + if modelName.hasPrefix("gemini-3") { + #expect(usageMetadata.thoughtsTokenCount == 64) + } else { + #expect(usageMetadata.thoughtsTokenCount == 0) + } // The fields `candidatesTokenCount` and `candidatesTokensDetails` are not included when using // Gemma models. - if modelName.hasPrefix("gemma") { + if modelName.hasPrefix("gemini-3") { + #expect(usageMetadata.candidatesTokenCount == 2) + #expect(usageMetadata.candidatesTokensDetails.isEmpty) + } else if modelName.hasPrefix("gemma") { #expect(usageMetadata.candidatesTokenCount == 0) #expect(usageMetadata.candidatesTokensDetails.isEmpty) } else { @@ -95,9 +104,12 @@ struct GenerateContentIntegrationTests { #expect(candidatesTokensDetails.modality == .text) #expect(candidatesTokensDetails.tokenCount == usageMetadata.candidatesTokenCount) } - #expect(usageMetadata.totalTokenCount > 0) - #expect(usageMetadata.totalTokenCount == - (usageMetadata.promptTokenCount + usageMetadata.candidatesTokenCount)) + if modelName.hasPrefix("gemini-3") { + #expect(usageMetadata.totalTokenCount == 80) + } else { + #expect(usageMetadata.totalTokenCount == + (usageMetadata.promptTokenCount + usageMetadata.candidatesTokenCount)) + } } @Test( @@ -161,16 +173,16 @@ struct GenerateContentIntegrationTests { (.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig( thinkingBudget: 32768, includeThoughts: true )), - (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 0)), - ( - .googleAI_v1beta_freeTier, - ModelNames.gemini2_5_Flash, - ThinkingConfig(thinkingBudget: 24576) - ), - (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig( - thinkingBudget: 24576, includeThoughts: true - )), // Note: The following configs are commented out for easy one-off manual testing. +// (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 0)), +// ( +// .googleAI_v1beta_freeTier, +// ModelNames.gemini2_5_Flash, +// ThinkingConfig(thinkingBudget: 24576) +// ), +// (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig( +// thinkingBudget: 24576, includeThoughts: true +// )), // (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( // thinkingBudget: 0 // )), @@ -475,14 +487,14 @@ struct GenerateContentIntegrationTests { (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), - (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), // Note: The following configs are commented out for easy one-off manual testing. // (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2FlashLite), // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemma3_4B), +// (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), +// (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), ]) func generateContentStream(_ config: InstanceConfig, modelName: String) async throws { let expectedResponse = [ diff --git a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift index 6522abbad1e..476be89b878 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift @@ -83,8 +83,8 @@ struct InstanceConfig: Equatable, Encodable { vertexAI_v1beta_global_appCheckLimitedUse, googleAI_v1beta, googleAI_v1beta_appCheckLimitedUse, - googleAI_v1beta_freeTier, // Note: The following configs are commented out for easy one-off manual testing. + // googleAI_v1beta_freeTier, // vertexAI_v1beta_staging, // vertexAI_v1beta_staging_global_bypassProxy, // googleAI_v1beta_staging, From 7cf020378e5f61146c4ee045dc80464acb6a8362 Mon Sep 17 00:00:00 2001 From: Paul Beusterien Date: Tue, 23 Dec 2025 07:05:19 -0800 Subject: [PATCH 2/6] More gemini 3 --- .../GenerateContentIntegrationTests.swift | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index ae214a2ff2c..f8102f70229 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -104,12 +104,9 @@ struct GenerateContentIntegrationTests { #expect(candidatesTokensDetails.modality == .text) #expect(candidatesTokensDetails.tokenCount == usageMetadata.candidatesTokenCount) } - if modelName.hasPrefix("gemini-3") { - #expect(usageMetadata.totalTokenCount == 80) - } else { - #expect(usageMetadata.totalTokenCount == - (usageMetadata.promptTokenCount + usageMetadata.candidatesTokenCount)) - } + #expect(usageMetadata.totalTokenCount == (usageMetadata.promptTokenCount + + usageMetadata.candidatesTokenCount + + usageMetadata.thoughtsTokenCount)) } @Test( @@ -173,6 +170,11 @@ struct GenerateContentIntegrationTests { (.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig( thinkingBudget: 32768, includeThoughts: true )), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig(thinkingBudget: 128)), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig(thinkingBudget: 32768)), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig( + thinkingBudget: 32768, includeThoughts: true + )), // Note: The following configs are commented out for easy one-off manual testing. // (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 0)), // ( @@ -267,6 +269,10 @@ struct GenerateContentIntegrationTests { (.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig( thinkingBudget: -1, includeThoughts: true )), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig(thinkingBudget: -1)), + (.googleAI_v1beta, ModelNames.gemini3FlashPreview, ThinkingConfig( + thinkingBudget: -1, includeThoughts: true + )), ] as [(InstanceConfig, String, ThinkingConfig)] ) func generateContentThinkingFunctionCalling(_ config: InstanceConfig, modelName: String, @@ -482,8 +488,8 @@ struct GenerateContentIntegrationTests { @Test(arguments: [ (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashLite), - (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashLite), - (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), + (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini3FlashPreview), + (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini3FlashPreview), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), From ae7f9c47a3f8e6dde44490f16b264134ad1d9064 Mon Sep 17 00:00:00 2001 From: Paul Beusterien Date: Tue, 23 Dec 2025 07:48:47 -0800 Subject: [PATCH 3/6] Better test disable --- .../FirebaseAITestApp.xcodeproj/project.pbxproj | 10 ---------- .../Tests/Integration/LiveSessionTests.swift | 17 +++++++++-------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj b/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj index 8b59429f1e5..c40918762a4 100644 --- a/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj +++ b/FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj/project.pbxproj @@ -36,13 +36,6 @@ ); target = 866138682CC943DE00F4B78E /* IntegrationTests-SPM */; }; - DECF58E22EFA338E000A3EC4 /* Exceptions for "Tests" folder in "IntegrationTests-SPM" target */ = { - isa = PBXFileSystemSynchronizedBuildFileExceptionSet; - membershipExceptions = ( - Integration/LiveSessionTests.swift, - ); - target = 866138682CC943DE00F4B78E /* IntegrationTests-SPM */; - }; /* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -53,9 +46,6 @@ }; 863E93F42EC69AB100BE4F4E /* Tests */ = { isa = PBXFileSystemSynchronizedRootGroup; - exceptions = ( - DECF58E22EFA338E000A3EC4 /* Exceptions for "Tests" folder in "IntegrationTests-SPM" target */, - ); path = Tests; sourceTree = ""; }; diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift index fecf8e80e7b..10fcb93aa47 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift @@ -91,7 +91,7 @@ struct LiveSessionTests { ) } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendTextRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -113,7 +113,7 @@ struct LiveSessionTests { #expect(modelResponse == "yes") } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendTextRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( @@ -136,7 +136,7 @@ struct LiveSessionTests { #expect(modelResponse == "yes") } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendAudioRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( @@ -165,7 +165,7 @@ struct LiveSessionTests { #expect(modelResponse == "goodbye") } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func sendAudioRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -192,7 +192,7 @@ struct LiveSessionTests { #expect(modelResponse == "goodbye") } - @Test(arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive }) + @Test(.disabled("Temporarily disabled"), arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive }) // gemini-2.0-flash-live-001 is buggy and likes to respond to the audio or system instruction // (eg; it will say 'okay' or 'hello', instead of following the instructions) func sendVideoRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { @@ -235,7 +235,7 @@ struct LiveSessionTests { #expect(["kitten", "cat", "kitty"].contains(modelResponse)) } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func realtime_functionCalling(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -283,7 +283,7 @@ struct LiveSessionTests { #expect(modelResponse == "smith") } - @Test(arguments: arguments.filter { + @Test(.disabled("Temporarily disabled"), arguments: arguments.filter { // TODO: (b/450982184) Remove when Vertex AI adds support for Function IDs and Cancellation switch $0.0.apiConfig.service { case .googleAI: @@ -325,6 +325,7 @@ struct LiveSessionTests { } @Test( + .disabled("Temporarily disabled"), arguments: arguments.filter { !$0.0.useLimitedUseAppCheckTokens } ) // Getting a limited use token adds too much of an overhead; we can't interrupt the model in time @@ -360,7 +361,7 @@ struct LiveSessionTests { } } - @Test(arguments: arguments) + @Test(.disabled("Temporarily disabled"), arguments: arguments) func incremental_works(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, From 47773bbdea8926b282809eab3e0e48c56b76b2ca Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 23 Dec 2025 10:54:03 -0500 Subject: [PATCH 4/6] Update Gemini Live model versions and re-enable some tests --- .../Tests/TestApp/Sources/Constants.swift | 5 +- .../Tests/Integration/LiveSessionTests.swift | 84 +------------------ 2 files changed, 5 insertions(+), 84 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Sources/Constants.swift b/FirebaseAI/Tests/TestApp/Sources/Constants.swift index 6ed79da5aae..3caa676bc11 100644 --- a/FirebaseAI/Tests/TestApp/Sources/Constants.swift +++ b/FirebaseAI/Tests/TestApp/Sources/Constants.swift @@ -23,12 +23,11 @@ public enum FirebaseAppNames { public enum ModelNames { public static let gemini2Flash = "gemini-2.0-flash-001" public static let gemini2FlashLite = "gemini-2.0-flash-lite-001" - public static let gemini2FlashLive = "gemini-2.0-flash-live-001" - public static let gemini2FlashLivePreview = "gemini-2.0-flash-live-preview-04-09" public static let gemini2_5_FlashImage = "gemini-2.5-flash-image" public static let gemini2_5_Flash = "gemini-2.5-flash" public static let gemini2_5_FlashLite = "gemini-2.5-flash-lite" - public static let gemini2_5_FlashLivePreview = "gemini-live-2.5-flash-preview" + public static let gemini2_5_FlashLive = "gemini-live-2.5-flash-native-audio" + public static let gemini2_5_FlashLivePreview = "gemini-2.5-flash-native-audio-preview-12-2025" public static let gemini2_5_Pro = "gemini-2.5-pro" public static let gemini3FlashPreview = "gemini-3-flash-preview" public static let gemma3_4B = "gemma-3-4b-it" diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift index 10fcb93aa47..4ef24c51569 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift @@ -25,11 +25,10 @@ struct LiveSessionTests { switch config.apiConfig.service { case .vertexAI: [ - (config, ModelNames.gemini2FlashLivePreview), + (config, ModelNames.gemini2_5_FlashLive), ] case .googleAI: [ - (config, ModelNames.gemini2FlashLive), (config, ModelNames.gemini2_5_FlashLivePreview), ] } @@ -91,52 +90,7 @@ struct LiveSessionTests { ) } - @Test(.disabled("Temporarily disabled"), arguments: arguments) - func sendTextRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { - let model = FirebaseAI.componentInstance(config).liveModel( - modelName: modelName, - generationConfig: textConfig, - systemInstruction: SystemInstructions.yesOrNo - ) - - let session = try await model.connect() - await session.sendTextRealtime("Does five plus five equal ten?") - - let text = try await session.collectNextTextResponse() - - await session.close() - let modelResponse = text - .trimmingCharacters(in: .whitespacesAndNewlines) - .trimmingCharacters(in: .punctuationCharacters) - .lowercased() - - #expect(modelResponse == "yes") - } - - @Test(.disabled("Temporarily disabled"), arguments: arguments) - func sendTextRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig, - modelName: String) async throws { - let model = FirebaseAI.componentInstance(config).liveModel( - modelName: modelName, - generationConfig: audioConfig, - systemInstruction: SystemInstructions.yesOrNo - ) - - let session = try await model.connect() - await session.sendTextRealtime("Does five plus five equal ten?") - - let text = try await session.collectNextAudioOutputTranscript() - - await session.close() - let modelResponse = text - .trimmingCharacters(in: .whitespacesAndNewlines) - .trimmingCharacters(in: .punctuationCharacters) - .lowercased() - - #expect(modelResponse == "yes") - } - - @Test(.disabled("Temporarily disabled"), arguments: arguments) + @Test(arguments: arguments) func sendAudioRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( @@ -166,35 +120,6 @@ struct LiveSessionTests { } @Test(.disabled("Temporarily disabled"), arguments: arguments) - func sendAudioRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { - let model = FirebaseAI.componentInstance(config).liveModel( - modelName: modelName, - generationConfig: textConfig, - systemInstruction: SystemInstructions.helloGoodbye - ) - - let session = try await model.connect() - - let audioFile = try #require( - NSDataAsset(name: "hello"), "Missing audio file 'hello.wav' in Assets" - ) - await session.sendAudioRealtime(audioFile.data) - await session.sendAudioRealtime(Data(repeating: 0, count: audioFile.data.count)) - - let text = try await session.collectNextTextResponse() - - await session.close() - let modelResponse = text - .trimmingCharacters(in: .whitespacesAndNewlines) - .trimmingCharacters(in: .punctuationCharacters) - .lowercased() - - #expect(modelResponse == "goodbye") - } - - @Test(.disabled("Temporarily disabled"), arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive }) - // gemini-2.0-flash-live-001 is buggy and likes to respond to the audio or system instruction - // (eg; it will say 'okay' or 'hello', instead of following the instructions) func sendVideoRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -324,10 +249,7 @@ struct LiveSessionTests { await session.close() } - @Test( - .disabled("Temporarily disabled"), - arguments: arguments.filter { !$0.0.useLimitedUseAppCheckTokens } - ) + @Test(arguments: arguments.filter { !$0.0.useLimitedUseAppCheckTokens }) // Getting a limited use token adds too much of an overhead; we can't interrupt the model in time func realtime_interruption(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( From 290395597d5f8ba6d43700ce1c1dc0948fb2f6d8 Mon Sep 17 00:00:00 2001 From: Paul Beusterien Date: Tue, 23 Dec 2025 08:30:03 -0800 Subject: [PATCH 5/6] style --- .../Tests/TestApp/Tests/Integration/LiveSessionTests.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift index 10fcb93aa47..7c661f205d0 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift @@ -192,7 +192,10 @@ struct LiveSessionTests { #expect(modelResponse == "goodbye") } - @Test(.disabled("Temporarily disabled"), arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive }) + @Test( + .disabled("Temporarily disabled"), + arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive } + ) // gemini-2.0-flash-live-001 is buggy and likes to respond to the audio or system instruction // (eg; it will say 'okay' or 'hello', instead of following the instructions) func sendVideoRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { From f93bcb1ab0c2ee26df9defd9be5c53f38e061228 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 23 Dec 2025 12:07:19 -0500 Subject: [PATCH 6/6] Add Swift Testing `.bug()` to disabled tests --- .../Tests/Integration/LiveSessionTests.swift | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift index 4ef24c51569..7e1ceeb5751 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift @@ -119,7 +119,11 @@ struct LiveSessionTests { #expect(modelResponse == "goodbye") } - @Test(.disabled("Temporarily disabled"), arguments: arguments) + @Test( + .disabled("Temporarily disabled"), + .bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"), + arguments: arguments + ) func sendVideoRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -160,7 +164,11 @@ struct LiveSessionTests { #expect(["kitten", "cat", "kitty"].contains(modelResponse)) } - @Test(.disabled("Temporarily disabled"), arguments: arguments) + @Test( + .disabled("Temporarily disabled"), + .bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"), + arguments: arguments + ) func realtime_functionCalling(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName, @@ -208,15 +216,19 @@ struct LiveSessionTests { #expect(modelResponse == "smith") } - @Test(.disabled("Temporarily disabled"), arguments: arguments.filter { - // TODO: (b/450982184) Remove when Vertex AI adds support for Function IDs and Cancellation - switch $0.0.apiConfig.service { - case .googleAI: - true - case .vertexAI: - false + @Test( + .disabled("Temporarily disabled"), + .bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"), + arguments: arguments.filter { + // TODO: (b/450982184) Remove when Vertex AI adds support for Function IDs and Cancellation + switch $0.0.apiConfig.service { + case .googleAI: + true + case .vertexAI: + false + } } - }) + ) func realtime_functionCalling_cancellation(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( @@ -283,7 +295,11 @@ struct LiveSessionTests { } } - @Test(.disabled("Temporarily disabled"), arguments: arguments) + @Test( + .disabled("Temporarily disabled"), + .bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"), + arguments: arguments + ) func incremental_works(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).liveModel( modelName: modelName,