-
Notifications
You must be signed in to change notification settings - Fork 3k
feat(stats): add model cost estimation + fix model priority #3631
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
179d1d3
eb83cb3
48022dc
82d21d2
d023b71
4a4dc03
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -105,7 +105,12 @@ describe('statsCommand', () => { | |
| it('should return error if sessionStartTime is not available', async () => { | ||
| if (!statsCommand.action) throw new Error('Command has no action'); | ||
|
|
||
| nonInteractiveContext.session.stats.sessionStartTime = undefined; | ||
| ( | ||
| nonInteractiveContext.session.stats as unknown as Record< | ||
| string, | ||
| unknown | ||
| > | ||
| )['sessionStartTime'] = undefined; | ||
|
|
||
| const result = (await statsCommand.action(nonInteractiveContext, '')) as { | ||
| type: string; | ||
|
|
@@ -145,5 +150,333 @@ describe('statsCommand', () => { | |
| expect(result.type).toBe('message'); | ||
| expect(nonInteractiveContext.ui.addItem).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
||
| it('stats model shows cost when pricing is configured', async () => { | ||
| const modelSubCommand = statsCommand.subCommands?.find( | ||
| (sc) => sc.name === 'model', | ||
| ); | ||
| if (!modelSubCommand?.action) throw new Error('Subcommand has no action'); | ||
|
|
||
| const contextWithPricing = createMockCommandContext({ | ||
| executionMode: 'non_interactive', | ||
| }); | ||
| // Set up settings with modelPricing | ||
| ( | ||
| contextWithPricing.services.settings as unknown as Record< | ||
| string, | ||
| unknown | ||
| > | ||
| )['merged'] = { | ||
| modelPricing: { | ||
| 'test-model': { | ||
| inputPerMillionTokens: 0.3, | ||
| outputPerMillionTokens: 1.2, | ||
| }, | ||
| }, | ||
| }; | ||
| // Set up model metrics | ||
| contextWithPricing.session.stats.metrics.models = { | ||
| 'test-model': { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Critical] These new — gpt-5.5 via Qwen Code /review |
||
| tokens: { | ||
| prompt: 1_000_000, | ||
| candidates: 500_000, | ||
| cached: 0, | ||
| total: 1_500_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Suggestion] The non-interactive — gpt-5.5 via Qwen Code /review |
||
| }, | ||
| api: { | ||
| totalRequests: 10, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Suggestion] All 7 new cost test cases use Please add at least one test case with — glm-5.1 via Qwen Code /review |
||
| const result = (await modelSubCommand.action(contextWithPricing, '')) as { | ||
| type: string; | ||
| content: string; | ||
| }; | ||
|
|
||
| expect(result.type).toBe('message'); | ||
| expect(result.content).toContain('test-model'); | ||
| expect(result.content).toContain('prompt=1000000'); | ||
| expect(result.content).toContain('Estimated cost: $0.9000'); | ||
| }); | ||
|
|
||
| it('stats model does not show cost when pricing is not configured', async () => { | ||
| const modelSubCommand = statsCommand.subCommands?.find( | ||
| (sc) => sc.name === 'model', | ||
| ); | ||
| if (!modelSubCommand?.action) throw new Error('Subcommand has no action'); | ||
|
|
||
| const contextWithoutPricing = createMockCommandContext({ | ||
| executionMode: 'non_interactive', | ||
| }); | ||
| // Set up model metrics without pricing | ||
| contextWithoutPricing.session.stats.metrics.models = { | ||
| 'test-model': { | ||
| tokens: { | ||
| prompt: 1_000_000, | ||
| candidates: 500_000, | ||
| cached: 0, | ||
| total: 1_500_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 10, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = (await modelSubCommand.action( | ||
| contextWithoutPricing, | ||
| '', | ||
| )) as { type: string; content: string }; | ||
|
|
||
| expect(result.type).toBe('message'); | ||
| expect(result.content).toContain('test-model'); | ||
| expect(result.content).not.toContain('Estimated cost'); | ||
| }); | ||
|
|
||
| it('stats model shows cost per model when multiple models have pricing', async () => { | ||
| const modelSubCommand = statsCommand.subCommands?.find( | ||
| (sc) => sc.name === 'model', | ||
| ); | ||
| if (!modelSubCommand?.action) throw new Error('Subcommand has no action'); | ||
|
|
||
| const context = createMockCommandContext({ | ||
| executionMode: 'non_interactive', | ||
| }); | ||
| // Set up settings with multiple model pricing | ||
| (context.services.settings as unknown as Record<string, unknown>)[ | ||
| 'merged' | ||
| ] = { | ||
| modelPricing: { | ||
| 'model-a': { | ||
| inputPerMillionTokens: 0.5, | ||
| outputPerMillionTokens: 1.5, | ||
| }, | ||
| 'model-b': { | ||
| inputPerMillionTokens: 0.1, | ||
| outputPerMillionTokens: 0.5, | ||
| }, | ||
| }, | ||
| }; | ||
| // Set up multiple model metrics | ||
| context.session.stats.metrics.models = { | ||
| 'model-a': { | ||
| tokens: { | ||
| prompt: 2_000_000, | ||
| candidates: 1_000_000, | ||
| cached: 0, | ||
| total: 3_000_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 20, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| 'model-b': { | ||
| tokens: { | ||
| prompt: 500_000, | ||
| candidates: 200_000, | ||
| cached: 0, | ||
| total: 700_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 5, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = (await modelSubCommand.action(context, '')) as { | ||
| type: string; | ||
| content: string; | ||
| }; | ||
|
|
||
| expect(result.type).toBe('message'); | ||
| expect(result.content).toContain('model-a'); | ||
| expect(result.content).toContain('model-b'); | ||
| // model-a: 2M * $0.50 + 1M * $1.50 = $1.00 + $1.50 = $2.50 | ||
| // model-b: 500K * $0.10 + 200K * $0.50 = $0.05 + $0.10 = $0.15 | ||
| expect(result.content).toContain('Estimated cost: $2.5000'); | ||
| expect(result.content).toContain('Estimated cost: $0.1500'); | ||
| }); | ||
|
|
||
| it('stats model shows cost only for models with pricing', async () => { | ||
| const modelSubCommand = statsCommand.subCommands?.find( | ||
| (sc) => sc.name === 'model', | ||
| ); | ||
| if (!modelSubCommand?.action) throw new Error('Subcommand has no action'); | ||
|
|
||
| const context = createMockCommandContext({ | ||
| executionMode: 'non_interactive', | ||
| }); | ||
| // Only model-a has pricing | ||
| (context.services.settings as unknown as Record<string, unknown>)[ | ||
| 'merged' | ||
| ] = { | ||
| modelPricing: { | ||
| 'model-a': { | ||
| inputPerMillionTokens: 0.3, | ||
| outputPerMillionTokens: 1.2, | ||
| }, | ||
| // model-b has no pricing | ||
| }, | ||
| }; | ||
| context.session.stats.metrics.models = { | ||
| 'model-a': { | ||
| tokens: { | ||
| prompt: 1_000_000, | ||
| candidates: 1_000_000, | ||
| cached: 0, | ||
| total: 2_000_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 10, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| 'model-b': { | ||
| tokens: { | ||
| prompt: 1_000_000, | ||
| candidates: 1_000_000, | ||
| cached: 0, | ||
| total: 2_000_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 10, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = (await modelSubCommand.action(context, '')) as { | ||
| type: string; | ||
| content: string; | ||
| }; | ||
|
|
||
| expect(result.type).toBe('message'); | ||
| // model-a has pricing | ||
| expect(result.content).toContain('model-a'); | ||
| // model-b has no pricing | ||
| expect(result.content).toContain('model-b'); | ||
| // Count occurrences of "Estimated cost" | ||
| const costMatches = result.content.match(/Estimated cost/g); | ||
| expect(costMatches).toBeTruthy(); | ||
| expect(costMatches!.length).toBe(1); | ||
| }); | ||
|
|
||
| it('stats model handles zero tokens with pricing', async () => { | ||
| const modelSubCommand = statsCommand.subCommands?.find( | ||
| (sc) => sc.name === 'model', | ||
| ); | ||
| if (!modelSubCommand?.action) throw new Error('Subcommand has no action'); | ||
|
|
||
| const context = createMockCommandContext({ | ||
| executionMode: 'non_interactive', | ||
| }); | ||
| (context.services.settings as unknown as Record<string, unknown>)[ | ||
| 'merged' | ||
| ] = { | ||
| modelPricing: { | ||
| 'test-model': { | ||
| inputPerMillionTokens: 0.3, | ||
| outputPerMillionTokens: 1.2, | ||
| }, | ||
| }, | ||
| }; | ||
| context.session.stats.metrics.models = { | ||
| 'test-model': { | ||
| tokens: { | ||
| prompt: 0, | ||
| candidates: 0, | ||
| cached: 0, | ||
| total: 0, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 0, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = (await modelSubCommand.action(context, '')) as { | ||
| type: string; | ||
| content: string; | ||
| }; | ||
|
|
||
| expect(result.type).toBe('message'); | ||
| expect(result.content).toContain('test-model'); | ||
| // Zero tokens mean zero cost, so no cost line should appear | ||
| expect(result.content).not.toContain('Estimated cost'); | ||
| }); | ||
|
|
||
| it('stats model handles partial pricing (input only)', async () => { | ||
| const modelSubCommand = statsCommand.subCommands?.find( | ||
| (sc) => sc.name === 'model', | ||
| ); | ||
| if (!modelSubCommand?.action) throw new Error('Subcommand has no action'); | ||
|
|
||
| const context = createMockCommandContext({ | ||
| executionMode: 'non_interactive', | ||
| }); | ||
| (context.services.settings as unknown as Record<string, unknown>)[ | ||
| 'merged' | ||
| ] = { | ||
| modelPricing: { | ||
| 'test-model': { | ||
| inputPerMillionTokens: 0.3, | ||
| // No output pricing | ||
| }, | ||
| }, | ||
| }; | ||
| context.session.stats.metrics.models = { | ||
| 'test-model': { | ||
| tokens: { | ||
| prompt: 1_000_000, | ||
| candidates: 1_000_000, | ||
| cached: 0, | ||
| total: 2_000_000, | ||
| thoughts: 0, | ||
| tool: 0, | ||
| }, | ||
| api: { | ||
| totalRequests: 10, | ||
| totalErrors: 0, | ||
| totalLatencyMs: 0, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = (await modelSubCommand.action(context, '')) as { | ||
| type: string; | ||
| content: string; | ||
| }; | ||
|
|
||
| expect(result.type).toBe('message'); | ||
| // 1M input tokens * $0.30/M = $0.30 | ||
| expect(result.content).toContain('Estimated cost: $0.3000'); | ||
| }); | ||
| }); | ||
| }); | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[Critical]
modelPricingis missingmergeStrategy: MergeStrategy.SHALLOW_MERGE.Without an explicit merge strategy, it defaults to
REPLACE. This means any project-levelmodelPricingcompletely wipes out global pricing configured in~/.qwen/settings.json. A user who set pricing for 5 models globally and adds pricing for 1 project-specific model will silently lose the other 5.Every other top-level
Record<string, ...>setting (mcpServers,channels) usesSHALLOW_MERGEfor exactly this reason.— glm-5.1 via Qwen Code /review