Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,12 @@
"model": "gemini-3.1-flash-lite"
}
},
"gemini-3.5-flash": {

Check warning on line 605 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"extends": "chat-base-3",
"modelConfig": {
"model": "gemini-3.5-flash"

Check warning on line 608 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
},
"gemma-4-31b-it": {
"extends": "chat-base-3",
"modelConfig": {
Expand All @@ -626,6 +632,12 @@
"model": "gemini-3-flash-preview"
}
},
"gemini-3.5-flash-base": {

Check warning on line 635 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"extends": "base",
"modelConfig": {
"model": "gemini-3.5-flash"

Check warning on line 638 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
},
"classifier": {
"extends": "base",
"modelConfig": {
Expand Down Expand Up @@ -868,6 +880,16 @@
"multimodalToolUse": true
}
},
"gemini-3.5-flash": {

Check warning on line 883 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"tier": "flash",
"family": "gemini-3",
"isPreview": false,
"isVisible": true,
"features": {
"thinking": false,
"multimodalToolUse": true
}
},
"gemini-2.5-pro": {
"tier": "pro",
"family": "gemini-2.5",
Expand Down Expand Up @@ -1020,9 +1042,44 @@
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1047 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
},
{
"condition": {
"hasAccessToPreview": false
},
"target": "gemini-2.5-flash"
}
]
},
"gemini-3.5-flash": {

Check warning on line 1057 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"default": "gemini-3.5-flash",

Check warning on line 1058 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"contexts": [
{
"condition": {
"useGemini3_5Flash": false,
"hasAccessToPreview": false
},
"target": "gemini-2.5-flash"
},
{
"condition": {
"useGemini3_5Flash": false
},
"target": "gemini-3-flash-preview"
}
]
},
"gemini-2.5-flash": {
"default": "gemini-2.5-flash",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1082 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
]
},
Expand Down Expand Up @@ -1104,6 +1161,12 @@
"flash": {
"default": "gemini-3-flash-preview",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1168 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
},
{
"condition": {
"hasAccessToPreview": false
Expand Down Expand Up @@ -1157,6 +1220,12 @@
"flash": {
"default": "gemini-3-flash-preview",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"
},
{
"condition": {
"hasAccessToPreview": false
Expand Down
28 changes: 24 additions & 4 deletions evals/llm-judge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,30 @@ export class LLMJudge {

for (const res of rawResults) {
// Remove any punctuation the model might have appended
const cleanRes = res.replace(/[^A-Z]/g, '');
if (cleanRes.startsWith('YES')) yes++;
else if (cleanRes.startsWith('NO')) no++;
else other++;
const cleanRes = res.replace(/[^A-Z ]/g, '');
if (
cleanRes.includes('THE ANSWER IS YES') ||
cleanRes.includes('ANSWER IS YES') ||
Comment thread
DavidAPierce marked this conversation as resolved.
Resolved
cleanRes.endsWith('YES')
) {
yes++;
} else if (
cleanRes.includes('THE ANSWER IS NO') ||
cleanRes.includes('ANSWER IS NO') ||
cleanRes.endsWith('NO')
) {
no++;
} else if (cleanRes.trim() === 'YES') {
yes++;
} else if (cleanRes.trim() === 'NO') {
no++;
} else {
// Fallback: look for YES or NO as standalone words or at the end
const words = cleanRes.split(/\s+/);
if (words.includes('YES')) yes++;
else if (words.includes('NO')) no++;
else other++;
}
}

// Pass if YES > NO and YES > OTHER (plurality)
Expand Down
4 changes: 4 additions & 0 deletions packages/core/src/availability/policyCatalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export interface ModelPolicyOptions {
useGemini31?: boolean;
useGemini31FlashLite?: boolean;
useCustomToolModel?: boolean;
useGemini3_5Flash?: boolean;
}

const DEFAULT_ACTIONS: ModelPolicyActionMap = {
Expand Down Expand Up @@ -94,6 +95,9 @@ export function getModelPolicyChain(
PREVIEW_GEMINI_MODEL,
options.useGemini31,
options.useCustomToolModel,
true,
undefined,
options.useGemini3_5Flash,
);
return [
definePolicy({
Expand Down
5 changes: 5 additions & 0 deletions packages/core/src/availability/policyHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export function resolvePolicyChain(
const useGemini31 = config.getGemini31LaunchedSync?.() ?? false;
const useCustomToolModel = config.getUseCustomToolModelSync?.() ?? false;
const hasAccessToPreview = config.getHasAccessToPreviewModel?.() ?? false;
const useGemini3_5Flash = config.hasGemini35FlashGAAccess?.() ?? false;

// Capture the original family intent before any normalization or early downgrade.
const isOriginallyGemini3 = isGemini3Model(modelFromConfig, config);
Expand All @@ -65,6 +66,7 @@ export function resolvePolicyChain(
useCustomToolModel,
hasAccessToPreview,
config,
useGemini3_5Flash,
),
);
const isAutoPreferred = normalizedPreferredModel
Expand All @@ -82,6 +84,7 @@ export function resolvePolicyChain(
const context = {
useGemini3_1: useGemini31,
useCustomTools: useCustomToolModel,
useGemini3_5Flash,
};

if (resolvedModel === DEFAULT_GEMINI_FLASH_LITE_MODEL) {
Expand Down Expand Up @@ -136,6 +139,7 @@ export function resolvePolicyChain(
userTier: config.getUserTier(),
useGemini31,
useCustomToolModel,
useGemini3_5Flash,
});
} else {
// User requested Gemini 3 but has no access. Proactively downgrade
Expand All @@ -146,6 +150,7 @@ export function resolvePolicyChain(
userTier: config.getUserTier(),
useGemini31,
useCustomToolModel,
useGemini3_5Flash,
});
}
} else {
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/code_assist/experiments/flagNames.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export const ExperimentFlags = {
GEMINI_3_1_PRO_LAUNCHED: 45760185,
PRO_MODEL_NO_ACCESS: 45768879,
DEFAULT_REQUEST_TIMEOUT: 45773134,
GEMINI_3_5_FLASH_GA_LAUNCHED: 45780819,
} as const;

export type ExperimentFlagName =
Expand Down
20 changes: 20 additions & 0 deletions packages/core/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2054,6 +2054,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);

const isPreview = isPreviewModel(primaryModel, this);
Expand Down Expand Up @@ -2093,6 +2094,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);
return this.modelQuotas.get(primaryModel)?.remaining;
}
Expand All @@ -2108,6 +2110,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);
return this.modelQuotas.get(primaryModel)?.limit;
}
Expand All @@ -2123,6 +2126,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);
return this.modelQuotas.get(primaryModel)?.resetTime;
}
Expand Down Expand Up @@ -3537,6 +3541,22 @@ export class Config implements McpContext, AgentLoopContext {
);
}

/**
* Returns whether Gemini 3.5 Flash GA has been launched.
*
* Note: This method should only be called after startup, once experiments have been loaded.
*/
hasGemini35FlashGAAccess(): boolean {
const authType = this.contentGeneratorConfig?.authType;
if (this.isGemini31LaunchedForAuthType(authType)) {
return true;
}
return (
this.experiments?.flags[ExperimentFlags.GEMINI_3_5_FLASH_GA_LAUNCHED]
?.boolValue ?? false
);
}

/**
* Returns whether Gemini 3.1 has been launched.
*
Expand Down
41 changes: 41 additions & 0 deletions packages/core/src/config/defaultModelConfigs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,12 @@ export const DEFAULT_MODEL_CONFIGS: ModelConfigServiceConfig = {
model: 'gemini-3.1-flash-lite',
},
},
'gemini-3.5-flash': {
extends: 'chat-base-3',
modelConfig: {
model: 'gemini-3.5-flash',
},
},
'gemma-4-31b-it': {
extends: 'chat-base-3',
modelConfig: {
Expand All @@ -139,6 +145,12 @@ export const DEFAULT_MODEL_CONFIGS: ModelConfigServiceConfig = {
model: 'gemini-3-flash-preview',
},
},
'gemini-3.5-flash-base': {
extends: 'base',
modelConfig: {
model: 'gemini-3.5-flash',
},
},
classifier: {
extends: 'base',
modelConfig: {
Expand Down Expand Up @@ -346,6 +358,13 @@ export const DEFAULT_MODEL_CONFIGS: ModelConfigServiceConfig = {
isVisible: true,
features: { thinking: false, multimodalToolUse: true },
},
'gemini-3.5-flash': {
tier: 'flash',
family: 'gemini-3',
isPreview: false,
isVisible: true,
features: { thinking: false, multimodalToolUse: true },
},
'gemini-2.5-pro': {
tier: 'pro',
family: 'gemini-2.5',
Expand Down Expand Up @@ -450,12 +469,32 @@ export const DEFAULT_MODEL_CONFIGS: ModelConfigServiceConfig = {
'gemini-3-flash-preview': {
default: 'gemini-3-flash-preview',
contexts: [
{ condition: { useGemini3_5Flash: true }, target: 'gemini-3.5-flash' },
{
condition: { hasAccessToPreview: false },
target: 'gemini-2.5-flash',
},
],
},
'gemini-3.5-flash': {
default: 'gemini-3.5-flash',
contexts: [
{
condition: { useGemini3_5Flash: false, hasAccessToPreview: false },
target: 'gemini-2.5-flash',
},
{
condition: { useGemini3_5Flash: false },
target: 'gemini-3-flash-preview',
},
],
},
Comment thread
DavidAPierce marked this conversation as resolved.
Resolved
'gemini-2.5-flash': {
default: 'gemini-2.5-flash',
contexts: [
{ condition: { useGemini3_5Flash: true }, target: 'gemini-3.5-flash' },
],
},
'gemini-3-pro-preview': {
default: 'gemini-3-pro-preview',
contexts: [
Expand Down Expand Up @@ -504,6 +543,7 @@ export const DEFAULT_MODEL_CONFIGS: ModelConfigServiceConfig = {
flash: {
default: 'gemini-3-flash-preview',
contexts: [
{ condition: { useGemini3_5Flash: true }, target: 'gemini-3.5-flash' },
{
condition: { hasAccessToPreview: false },
target: 'gemini-2.5-flash',
Expand Down Expand Up @@ -535,6 +575,7 @@ export const DEFAULT_MODEL_CONFIGS: ModelConfigServiceConfig = {
flash: {
default: 'gemini-3-flash-preview',
contexts: [
{ condition: { useGemini3_5Flash: true }, target: 'gemini-3.5-flash' },
{
condition: { hasAccessToPreview: false },
target: 'gemini-2.5-flash',
Expand Down
Loading
Loading