Skip to content

fix: complete channel model batch testing #5532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Calcium-Ion merged 1 commit into from
Jun 17, 2026
Merged

fix: complete channel model batch testing #5532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
237 changes: 210 additions & 27 deletions web/default/src/features/channels/components/dialogs/channel-test-dialog.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.

For commercial licensing, please contact support@quantumnous.com
*/
import { type ChangeEvent, useCallback, useMemo, useState } from 'react'
import { type ChangeEvent, useCallback, useMemo, useRef, useState } from 'react'
import { useQueryClient } from '@tanstack/react-query'
import {
type ColumnDef,
Expand All @@ -32,6 +32,7 @@ import { Button } from '@/components/ui/button'
import { Checkbox } from '@/components/ui/checkbox'
import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label'
import { Progress } from '@/components/ui/progress'
import {
Select,
SelectContent,
Expand Down Expand Up @@ -103,6 +104,13 @@ type TestResult = {
errorCode?: string
}

type BatchProgress = {
total: number
completed: number
success: number
failed: number
}

type ChannelTestCachePatch = {
responseTime: number
testTime: number
Expand Down Expand Up @@ -181,6 +189,8 @@ const STREAM_INCOMPATIBLE_ENDPOINTS = new Set([

const MODEL_PRICE_ERROR_CODE = 'model_price_error'
const FAILURE_SUMMARY_MAX_LENGTH = 96
const BATCH_TEST_CONCURRENCY = 5
const BATCH_TEST_DELAY_MS = 100

type FailureStatusDisplay = {
summary: string
Expand All @@ -193,6 +203,10 @@ type FailureDetailsState = {
details: string
}

function sleep(ms: number) {
return new Promise<void>((resolve) => window.setTimeout(resolve, ms))
}

function normalizeInlineError(errorText: string) {
return errorText.replace(/\s+/g, ' ').trim()
}
Expand Down Expand Up @@ -289,6 +303,7 @@ function ChannelTestDialogContent({
const { t } = useTranslation()
const queryClient = useQueryClient()
const currentChannelId = currentRow.id
const batchStopRequestedRef = useRef(false)
const [endpointType, setEndpointType] = useState('auto')
const [isStreamTest, setIsStreamTest] = useState(false)
const [searchTerm, setSearchTerm] = useState('')
Expand All @@ -298,6 +313,8 @@ function ChannelTestDialogContent({
() => new Set()
)
const [isBatchTesting, setIsBatchTesting] = useState(false)
const [isBatchStopRequested, setIsBatchStopRequested] = useState(false)
const [batchProgress, setBatchProgress] = useState<BatchProgress | null>(null)
const [failureDetails, setFailureDetails] =
useState<FailureDetailsState | null>(null)
const [pagination, setPagination] = useState({
Expand All @@ -314,13 +331,16 @@ function ChannelTestDialogContent({
)

const resetState = useCallback(() => {
batchStopRequestedRef.current = true
setEndpointType('auto')
setIsStreamTest(false)
setSearchTerm('')
setTestResults({})
setRowSelection({})
setTestingModels(() => new Set())
setIsBatchTesting(false)
setIsBatchStopRequested(false)
setBatchProgress(null)
setFailureDetails(null)
setPagination({ pageIndex: 0, pageSize: 10 })
}, [])
Expand Down Expand Up @@ -501,29 +521,118 @@ function ChannelTestDialogContent({
]
)

const handleStopBatchTest = useCallback(() => {
if (!isBatchTesting || isBatchStopRequested) return

batchStopRequestedRef.current = true
setIsBatchStopRequested(true)
}, [isBatchStopRequested, isBatchTesting])

const handleBatchTest = useCallback(
async (modelsToTest: string[]) => {
if (!modelsToTest.length) return
const uniqueModels = Array.from(
new Set(modelsToTest.map((model) => model.trim()).filter(Boolean))
)
if (!uniqueModels.length) return

batchStopRequestedRef.current = false
setIsBatchTesting(true)
setIsBatchStopRequested(false)
setBatchProgress({
total: uniqueModels.length,
completed: 0,
success: 0,
failed: 0,
})

let resultPatch: ChannelTestCachePatch | undefined
const results: TestResult[] = []
let completedCount = 0
let successCount = 0
let failedCount = 0

try {
const settled = await Promise.allSettled(
modelsToTest.map((modelName) =>
testSingleModel(modelName, true, false)
)
)
const results = settled
.map((result) =>
result.status === 'fulfilled' ? result.value : undefined
const createFallbackResult = (error?: unknown): TestResult => ({
status: 'error',
completedAt: Date.now(),
error: error instanceof Error ? error.message : t('Test failed'),
})

const recordBatchResult = (result: TestResult) => {
results.push(result)
completedCount += 1
if (result.status === 'success') {
successCount += 1
}
failedCount = completedCount - successCount

setBatchProgress({
total: uniqueModels.length,
completed: completedCount,
success: successCount,
failed: failedCount,
})
}

for (
let startIndex = 0;
startIndex < uniqueModels.length;
startIndex += BATCH_TEST_CONCURRENCY
) {
if (batchStopRequestedRef.current) {
break
}

const batch = uniqueModels.slice(
startIndex,
startIndex + BATCH_TEST_CONCURRENCY
)
.filter((result): result is TestResult => Boolean(result))
const batchPromises = batch.map(async (modelName) => {
try {
const result = await testSingleModel(modelName, true, false)
const finalResult = result ?? createFallbackResult()
if (!result) {
updateTestResult(modelName, finalResult)
}
recordBatchResult(finalResult)
return finalResult
} catch (error: unknown) {
const fallbackResult = createFallbackResult(error)
updateTestResult(modelName, fallbackResult)
recordBatchResult(fallbackResult)
return fallbackResult
}
})

await Promise.allSettled(batchPromises)

if (
batchStopRequestedRef.current ||
startIndex + BATCH_TEST_CONCURRENCY >= uniqueModels.length
) {
break
}

await sleep(BATCH_TEST_DELAY_MS)
}

resultPatch = getLatestChannelTestCachePatch(results)
const successCount = results.filter(
(result) => result.status === 'success'
).length
const failedCount = modelsToTest.length - successCount
if (failedCount > 0) {
const stopped =
batchStopRequestedRef.current && completedCount < uniqueModels.length

if (stopped) {
toast.info(
t(
'Batch test stopped: {{completed}}/{{total}} completed, {{success}} succeeded, {{failed}} failed',
{
completed: completedCount,
total: uniqueModels.length,
success: successCount,
failed: failedCount,
}
)
)
} else if (failedCount > 0) {
toast.error(
t(
'Batch test completed: {{success}} succeeded, {{failed}} failed',
Expand All @@ -541,12 +650,15 @@ function ChannelTestDialogContent({
)
}
} finally {
batchStopRequestedRef.current = false
setIsBatchTesting(false)
setIsBatchStopRequested(false)
setBatchProgress(null)
setRowSelection({})
refreshChannelLists(resultPatch)
}
},
[refreshChannelLists, t, testSingleModel]
[refreshChannelLists, t, testSingleModel, updateTestResult]
)

const handleClose = useCallback(() => {
Expand All @@ -564,18 +676,22 @@ function ChannelTestDialogContent({
)

const isAnyTesting = testingModels.size > 0 || isBatchTesting
const isFilteringModels = searchTerm.trim().length > 0
const testAllButtonLabel = isFilteringModels
? t('Test {{count}} matching models', { count: filteredModels.length })
: t('Test all {{count}} models', { count: filteredModels.length })

const columns = useMemo<ColumnDef<ModelRow>[]>(
() => [
{
id: 'select',
header: ({ table }) => (
<Checkbox
checked={table.getIsAllPageRowsSelected()}
indeterminate={table.getIsSomePageRowsSelected()}
onCheckedChange={(value) =>
table.toggleAllPageRowsSelected(!!value)
checked={table.getIsAllRowsSelected()}
indeterminate={
table.getIsSomeRowsSelected() && !table.getIsAllRowsSelected()
}
onCheckedChange={(value) => table.toggleAllRowsSelected(!!value)}
aria-label={t('Select all models')}
/>
),
Expand Down Expand Up @@ -674,6 +790,7 @@ function ChannelTestDialogContent({
rowSelection,
pagination,
enableRowSelection: true,
getRowId: (row) => row.model,
onRowSelectionChange: setRowSelection,
onPaginationChange: setPagination,
withFilteredRowModel: false,
Expand Down Expand Up @@ -759,14 +876,41 @@ function ChannelTestDialogContent({
{t('Select models to run batch tests.')}
</p>
</div>
<Input
placeholder={t('Filter models...')}
value={searchTerm}
onChange={handleSearchTermChange}
className='sm:w-64'
/>
<div className='flex flex-col gap-2 sm:flex-row sm:items-center'>
<Input
placeholder={t('Filter models...')}
value={searchTerm}
onChange={handleSearchTermChange}
className='sm:w-64'
/>
{isBatchTesting ? (
<Button
variant='outline'
onClick={handleStopBatchTest}
disabled={isBatchStopRequested}
>
{isBatchStopRequested
? t('Stopping...')
: t('Stop testing')}
</Button>
) : (
<Button
onClick={() => handleBatchTest(filteredModels)}
disabled={isAnyTesting || filteredModels.length === 0}
>
{testAllButtonLabel}
</Button>
)}
</div>
</div>

{batchProgress && (
<BatchProgressSummary
progress={batchProgress}
isStopping={isBatchStopRequested}
/>
)}

<div className='space-y-3'>
<DataTableView
table={table}
Expand Down Expand Up @@ -827,6 +971,45 @@ function ChannelTestDialogContent({
)
}

function BatchProgressSummary({
progress,
isStopping,
}: {
progress: BatchProgress
isStopping: boolean
}) {
const { t } = useTranslation()
const progressValue =
progress.total > 0
? Math.min(100, Math.round((progress.completed / progress.total) * 100))
: 0

return (
<div className='bg-muted/30 flex flex-col gap-2 rounded-md border p-3'>
<div className='flex flex-col gap-1 sm:flex-row sm:items-center sm:justify-between'>
<p className='text-sm font-medium'>
{isStopping
? t('Stopping batch test...')
: t('Batch testing models...')}
</p>
<p className='text-muted-foreground text-xs tabular-nums'>
{t('{{completed}}/{{total}} completed', {
completed: progress.completed,
total: progress.total,
})}
</p>
</div>
<Progress value={progressValue} />
<p className='text-muted-foreground text-xs'>
{t('{{success}} succeeded, {{failed}} failed', {
success: progress.success,
failed: progress.failed,
})}
</p>
</div>
)
}

function TestStatusCell({
result,
model,
Expand Down
Loading