Skip to content

[AI] Update Gemini Live models in integration tests #15641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrewheard merged 8 commits into from
Dec 23, 2025
Merged

[AI] Update Gemini Live models in integration tests #15641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
47992c8
[AI] Make Integration tests green
paulb777 Dec 23, 2025
7cf0203
More gemini 3
paulb777 Dec 23, 2025
ae7f9c4
Better test disable
paulb777 Dec 23, 2025
47773bb
Update Gemini Live model versions and re-enable some tests
andrewheard Dec 23, 2025
2903955
style
paulb777 Dec 23, 2025
13cbbfb
Merge remote-tracking branch 'origin/pb-green-tests-again' into ah/pb…
andrewheard Dec 23, 2025
f93bcb1
Add Swift Testing `.bug()` to disabled tests
andrewheard Dec 23, 2025
56f4976
Merge branch 'main' into ah/pb-green-tests-again
andrewheard Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions FirebaseAI/Tests/TestApp/Sources/Constants.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,11 @@ public enum FirebaseAppNames {
public enum ModelNames {
public static let gemini2Flash = "gemini-2.0-flash-001"
public static let gemini2FlashLite = "gemini-2.0-flash-lite-001"
public static let gemini2FlashLive = "gemini-2.0-flash-live-001"
public static let gemini2FlashLivePreview = "gemini-2.0-flash-live-preview-04-09"
public static let gemini2_5_FlashImage = "gemini-2.5-flash-image"
public static let gemini2_5_Flash = "gemini-2.5-flash"
public static let gemini2_5_FlashLite = "gemini-2.5-flash-lite"
public static let gemini2_5_FlashLivePreview = "gemini-live-2.5-flash-preview"
public static let gemini2_5_FlashLive = "gemini-live-2.5-flash-native-audio"
public static let gemini2_5_FlashLivePreview = "gemini-2.5-flash-native-audio-preview-12-2025"
public static let gemini2_5_Pro = "gemini-2.5-pro"
public static let gemini3FlashPreview = "gemini-3-flash-preview"
public static let gemma3_4B = "gemma-3-4b-it"
Expand Down
119 changes: 27 additions & 92 deletions FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,10 @@ struct LiveSessionTests {
switch config.apiConfig.service {
case .vertexAI:
[
(config, ModelNames.gemini2FlashLivePreview),
(config, ModelNames.gemini2_5_FlashLive),
]
case .googleAI:
[
(config, ModelNames.gemini2FlashLive),
(config, ModelNames.gemini2_5_FlashLivePreview),
]
}
Expand Down Expand Up @@ -91,52 +90,7 @@ struct LiveSessionTests {
)
}

@Test(.disabled("Temporarily disabled"), arguments: arguments)
func sendTextRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
modelName: modelName,
generationConfig: textConfig,
systemInstruction: SystemInstructions.yesOrNo
)

let session = try await model.connect()
await session.sendTextRealtime("Does five plus five equal ten?")

let text = try await session.collectNextTextResponse()

await session.close()
let modelResponse = text
.trimmingCharacters(in: .whitespacesAndNewlines)
.trimmingCharacters(in: .punctuationCharacters)
.lowercased()

#expect(modelResponse == "yes")
}

@Test(.disabled("Temporarily disabled"), arguments: arguments)
func sendTextRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig,
modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
modelName: modelName,
generationConfig: audioConfig,
systemInstruction: SystemInstructions.yesOrNo
)

let session = try await model.connect()
await session.sendTextRealtime("Does five plus five equal ten?")

let text = try await session.collectNextAudioOutputTranscript()

await session.close()
let modelResponse = text
.trimmingCharacters(in: .whitespacesAndNewlines)
.trimmingCharacters(in: .punctuationCharacters)
.lowercased()

#expect(modelResponse == "yes")
}

@Test(.disabled("Temporarily disabled"), arguments: arguments)
@Test(arguments: arguments)
func sendAudioRealtime_receiveAudioOutputTranscripts(_ config: InstanceConfig,
modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
Expand Down Expand Up @@ -165,39 +119,11 @@ struct LiveSessionTests {
#expect(modelResponse == "goodbye")
}

@Test(.disabled("Temporarily disabled"), arguments: arguments)
func sendAudioRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
modelName: modelName,
generationConfig: textConfig,
systemInstruction: SystemInstructions.helloGoodbye
)

let session = try await model.connect()

let audioFile = try #require(
NSDataAsset(name: "hello"), "Missing audio file 'hello.wav' in Assets"
)
await session.sendAudioRealtime(audioFile.data)
await session.sendAudioRealtime(Data(repeating: 0, count: audioFile.data.count))

let text = try await session.collectNextTextResponse()

await session.close()
let modelResponse = text
.trimmingCharacters(in: .whitespacesAndNewlines)
.trimmingCharacters(in: .punctuationCharacters)
.lowercased()

#expect(modelResponse == "goodbye")
}

@Test(
.disabled("Temporarily disabled"),
arguments: arguments.filter { $0.1 != ModelNames.gemini2FlashLive }
.bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"),
arguments: arguments
)
// gemini-2.0-flash-live-001 is buggy and likes to respond to the audio or system instruction
// (eg; it will say 'okay' or 'hello', instead of following the instructions)
func sendVideoRealtime_receiveText(_ config: InstanceConfig, modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
modelName: modelName,
Expand Down Expand Up @@ -238,7 +164,11 @@ struct LiveSessionTests {
#expect(["kitten", "cat", "kitty"].contains(modelResponse))
}

@Test(.disabled("Temporarily disabled"), arguments: arguments)
@Test(
.disabled("Temporarily disabled"),
.bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"),
arguments: arguments
)
func realtime_functionCalling(_ config: InstanceConfig, modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
modelName: modelName,
Expand Down Expand Up @@ -286,15 +216,19 @@ struct LiveSessionTests {
#expect(modelResponse == "smith")
}

@Test(.disabled("Temporarily disabled"), arguments: arguments.filter {
// TODO: (b/450982184) Remove when Vertex AI adds support for Function IDs and Cancellation
switch $0.0.apiConfig.service {
case .googleAI:
true
case .vertexAI:
false
@Test(
.disabled("Temporarily disabled"),
.bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"),
arguments: arguments.filter {
// TODO: (b/450982184) Remove when Vertex AI adds support for Function IDs and Cancellation
switch $0.0.apiConfig.service {
case .googleAI:
true
case .vertexAI:
false
}
}
})
)
func realtime_functionCalling_cancellation(_ config: InstanceConfig,
modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
Expand Down Expand Up @@ -327,10 +261,7 @@ struct LiveSessionTests {
await session.close()
}

@Test(
.disabled("Temporarily disabled"),
arguments: arguments.filter { !$0.0.useLimitedUseAppCheckTokens }
)
@Test(arguments: arguments.filter { !$0.0.useLimitedUseAppCheckTokens })
// Getting a limited use token adds too much of an overhead; we can't interrupt the model in time
func realtime_interruption(_ config: InstanceConfig, modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
Expand Down Expand Up @@ -364,7 +295,11 @@ struct LiveSessionTests {
}
}

@Test(.disabled("Temporarily disabled"), arguments: arguments)
@Test(
.disabled("Temporarily disabled"),
.bug("https://github.com/firebase/firebase-ios-sdk/issues/15640"),
arguments: arguments
)
func incremental_works(_ config: InstanceConfig, modelName: String) async throws {
let model = FirebaseAI.componentInstance(config).liveModel(
modelName: modelName,
Expand Down
Loading