From bc2ba7bf6b01c7f50783d9505cb6c13aad784df4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 16 Aug 2026 15:03:11 -0500 Subject: [PATCH 01/10] feat(agentx): display TensorRT-LLM server metrics --- .../db/src/etl/compute-aggregate-stats.ts | 4 + .../db/src/etl/compute-chart-series.test.ts | 77 +++++++++++++ packages/db/src/etl/compute-chart-series.ts | 101 ++++++++++++++++-- .../db/src/etl/server-metrics-adapters.ts | 33 +++++- .../db/src/queries/agentic-aggregates.test.ts | 41 +++++++ packages/db/src/queries/agentic-aggregates.ts | 16 +++ 6 files changed, 263 insertions(+), 9 deletions(-) diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 07729b59b..54a896441 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -193,6 +193,10 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([ 'vllm:prefix_cache_queries', 'vllm:gpu_prefix_cache_hits', 'vllm:gpu_prefix_cache_queries', + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens_total', ]); /** diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 54295cc77..4b93ae0ea 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -146,6 +146,19 @@ function kvBlob(profiling: unknown[], warmup: unknown[] = []) { ); } +function buildTrtllmSeries( + endpoint_url: string, + dynamo_component: 'prefill' | 'backend', + value: number, + field: 'rate' | 'avg', +) { + return { + endpoint_url, + labels: { dynamo_component, worker_id: `${dynamo_component}-worker` }, + timeslices: [{ start_ns: 0, end_ns: 1e9, [field]: value }], + }; +} + describe('computeChartSeries', () => { it('returns null when the blob is null', async () => { expect(await computeChartSeries(null)).toBeNull(); @@ -708,6 +721,70 @@ describe('computeChartSeries', () => { expect(result?.metricSources).toEqual([]); }); + + it('extracts native TensorRT-LLM metrics and preserves disaggregated worker roles', async () => { + const prefillUrl = 'http://prefill-a.internal.test:7500/metrics'; + const decodeUrl = 'http://decode-a.internal.test:7501/metrics'; + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 0.3, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 0.7, 'avg'), + ], + }, + trtllm_kv_cache_host_utilization: { + series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], + }, + trtllm_prompt_tokens_total: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'), + buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'), + ], + }, + trtllm_prompt_cached_tokens_total: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'), + buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'), + ], + }, + trtllm_generation_tokens_total: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')], + }, + trtllm_num_requests_running: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 2, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 3, 'avg'), + ], + }, + trtllm_num_requests_waiting: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 4, 'avg')], + }, + }, + }); + + const result = await computeChartSeries(gzipSync(Buffer.from(json)), { + framework: 'trtllm', + disagg: true, + }); + + expect(result?.kvCacheUsage).toEqual([{ t: 0, value: 0.5 }]); + expect(result?.hostKvCacheUsage).toEqual([{ t: 0, value: 0.25 }]); + expect(result?.prefixCacheHitRate).toEqual([{ t: 0, value: 0.4 }]); + expect(result?.queueDepth).toEqual([{ t: 0, running: 5, waiting: 4, total: 9 }]); + expect(result?.prefillTps).toEqual([{ t: 0, value: 300 }]); + expect(result?.decodeTps).toEqual([{ t: 0, value: 50 }]); + expect(result?.promptTokensBySource).toEqual({ + 'cache hit (HBM)': [{ t: 0, value: 120 }], + 'compute (miss)': [{ t: 0, value: 180 }], + }); + expect(result?.metricSources.map(({ source }) => [source.role, source.endpointUrl])).toEqual([ + ['prefill', prefillUrl], + ['decode', decodeUrl], + ]); + expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]); + expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]); + }); }); // ── Summed series on the canonical grid (v14) ─────────────────────────── diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 82878ad86..39d4aac61 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -120,8 +120,10 @@ import { * per-rank detail stays reachable as that source's own * `kvCacheUsageByEngine`. * + * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV + * metrics, including per-prefill/decode source series for Dynamo disaggregation. */ -export const CHART_SERIES_VERSION = 15; +export const CHART_SERIES_VERSION = 16; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -244,6 +246,15 @@ export const CHART_METRIC_KEYS = new Set([ 'sglang:realtime_tokens', 'sglang:hicache_host_used_tokens', 'sglang:hicache_host_total_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_host_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens_total', + 'trtllm_generation_tokens_total', + 'trtllm_num_requests_running', + 'trtllm_num_requests_waiting', ]); /** @@ -976,6 +987,7 @@ function buildSeriesFromMetrics( 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); // One entry per logical engine (v13) — mirrored API-server frontends and the // warmup/profiling phase split are collapsed here rather than showing up as @@ -989,11 +1001,16 @@ function buildSeriesFromMetrics( // Prefix cache hit rate per scrape: Σhits.rate / Σqueries.rate across // engines, joined on start_ns. SGLang names: cached_tokens / prompt_tokens. - const hitsSeries = pickSeries('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const hitsSeries = pickSeries( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); const qsSeries = pickSeries( 'vllm:prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS); const qsOnGrid = summedSeries(qsSeries, tOf, 'rate', tickS); @@ -1003,10 +1020,25 @@ function buildSeriesFromMetrics( const q = qsByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q }); } + if (prefixCacheHitRate.length === 0) { + for (const [t, value] of sortedEntries( + aggregateByStart(metrics['trtllm_kv_cache_hit_rate']?.series, 'avg', 'avg'), + )) { + prefixCacheHitRate.push({ t: tOf(t), value }); + } + } // Queue depth: sum running + waiting across engines per timeslice. - const runSeries = pickSeries('vllm:num_requests_running', 'sglang:num_running_reqs'); - const waitSeries = pickSeries('vllm:num_requests_waiting', 'sglang:num_queue_reqs'); + const runSeries = pickSeries( + 'vllm:num_requests_running', + 'sglang:num_running_reqs', + 'trtllm_num_requests_running', + ); + const waitSeries = pickSeries( + 'vllm:num_requests_waiting', + 'sglang:num_queue_reqs', + 'trtllm_num_requests_waiting', + ); const runOnGrid = summedSeries(runSeries, tOf, 'avg', tickS); const waitByT = byT(summedSeries(waitSeries, tOf, 'avg', tickS)); const runByT = byT(runOnGrid); @@ -1025,11 +1057,23 @@ function buildSeriesFromMetrics( // work. const counterRate = (...names: string[]): TimeSeriesPoint[] => summedSeries(pickSeries(...names), tOf, 'rate', tickS); - const prefillTps = counterRate('vllm:prompt_tokens', 'sglang:prompt_tokens'); - const decodeTps = counterRate('vllm:generation_tokens', 'sglang:generation_tokens'); + const prefillTps = counterRate( + 'vllm:prompt_tokens', + 'sglang:prompt_tokens', + 'trtllm_prompt_tokens_total', + ); + const decodeTps = counterRate( + 'vllm:generation_tokens', + 'sglang:generation_tokens', + 'trtllm_generation_tokens_total', + ); // Tokens served from prefix cache per scrape. Lets the frontend derive // "cumulative unique input tokens served" = cumsum(prefillTps) − cumsum(hits). - const prefixCacheHitsTps = counterRate('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const prefixCacheHitsTps = counterRate( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); // SGLang hicache: host-pool KV cache utilization as used/total per // timeslice. Both metrics are gauges in absolute tokens. Total stays @@ -1049,6 +1093,13 @@ function buildSeriesFromMetrics( hostKvCacheUsage.push({ t, value: used / total }); } } + if (hostKvCacheUsage.length === 0) { + for (const [t, value] of sortedEntries( + aggregateByStart(metrics['trtllm_kv_cache_host_utilization']?.series, 'avg', 'avg'), + )) { + hostKvCacheUsage.push({ t: tOf(t), value }); + } + } // Per-source prompt tokens — sum across engines per source label. // vllm: vllm:prompt_tokens_by_source has one series per source label @@ -1108,6 +1159,27 @@ function buildSeriesFromMetrics( addSeriesRates(label, series); } } + if (promptBySrcByT.size === 0) { + const promptByT = aggregateByStart( + metrics['trtllm_prompt_tokens_total']?.series, + 'rate', + 'sum', + ); + const cachedByT = aggregateByStart( + metrics['trtllm_prompt_cached_tokens_total']?.series, + 'rate', + 'sum', + ); + const cachedSeries: RawSeries = { timeslices: [] }; + const computedSeries: RawSeries = { timeslices: [] }; + for (const [t, prompt] of promptByT) { + const cached = Math.max(0, cachedByT.get(t) ?? 0); + cachedSeries.timeslices!.push({ start_ns: t, rate: cached }); + computedSeries.timeslices!.push({ start_ns: t, rate: Math.max(0, prompt - cached) }); + } + addSeriesRates('cache hit (HBM)', cachedSeries); + addSeriesRates('compute (miss)', computedSeries); + } const promptTokensBySource: Record = {}; for (const [source, seriesForSource] of promptBySrc) { // Idle ticks are dropped rather than emitted as zeros: this feeds a @@ -1119,10 +1191,23 @@ function buildSeriesFromMetrics( const metricSources: MetricSourceSeries[] = []; const adapter = selectServerMetricsAdapter(context); if (includeMetricSources && context.disagg && adapter.id !== 'generic') { + const endpointRoles = new Map(); + for (const metric of Object.values(metrics)) { + for (const series of metric.series ?? []) { + const endpointUrl = series.endpoint_url; + const role = series.labels?.['disaggregation_mode'] ?? series.labels?.['dynamo_component']; + if (endpointUrl && role) endpointRoles.set(endpointUrl, role); + } + } const grouped = new Map(); for (const [metricName, metric] of Object.entries(metrics)) { for (const series of metric.series ?? []) { - const source = adapter.identifySource(series); + const roleHint = series.endpoint_url ? endpointRoles.get(series.endpoint_url) : undefined; + const identifiedSeries = + roleHint && !series.labels?.['disaggregation_mode'] + ? { ...series, labels: { ...series.labels, disaggregation_mode: roleHint } } + : series; + const source = adapter.identifySource(identifiedSeries); let group = grouped.get(source.id); if (!group) { group = { source, metrics: {} }; diff --git a/packages/db/src/etl/server-metrics-adapters.ts b/packages/db/src/etl/server-metrics-adapters.ts index 5e80e8bd6..45f3bacc1 100644 --- a/packages/db/src/etl/server-metrics-adapters.ts +++ b/packages/db/src/etl/server-metrics-adapters.ts @@ -78,6 +78,37 @@ const dynamoAdapter: ServerMetricsAdapter = { }, }; +const trtllmAdapter: ServerMetricsAdapter = { + id: 'trtllm', + matches: ({ framework }) => framework?.toLowerCase().includes('trt') ?? false, + identifySource(series) { + const labels = series.labels ?? {}; + const nativeRole = labels['disaggregation_mode'] ?? labels['dynamo_component'] ?? null; + const role: MetricSourceRole = + nativeRole === 'prefill' + ? 'prefill' + : nativeRole === 'decode' || nativeRole === 'backend' + ? 'decode' + : nativeRole === 'aggregated' + ? 'combined' + : 'unknown'; + const endpointUrl = series.endpoint_url ?? null; + const workerId = labels['worker_id'] ?? null; + const dpRank = labels['dp_rank'] ?? null; + const engine = labels['engine'] ?? labels['engine_idx'] ?? null; + return { + id: stableId('trtllm', [role, endpointUrl, workerId, dpRank, engine]), + adapter: 'trtllm', + role, + endpointUrl, + nativeRole, + workerId, + dpRank, + engine, + }; + }, +}; + const genericAdapter: ServerMetricsAdapter = { id: 'generic', matches: () => true, @@ -100,7 +131,7 @@ const genericAdapter: ServerMetricsAdapter = { }, }; -const ADAPTERS: readonly ServerMetricsAdapter[] = [dynamoAdapter, genericAdapter]; +const ADAPTERS: readonly ServerMetricsAdapter[] = [trtllmAdapter, dynamoAdapter, genericAdapter]; export function selectServerMetricsAdapter(context: ServerMetricsContext): ServerMetricsAdapter { return ADAPTERS.find((adapter) => adapter.matches(context)) ?? genericAdapter; diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 89ec4b914..7882c0741 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,6 +129,47 @@ describe('extractServerMetricSamples', () => { expect(out.kvCacheUtil).toEqual([]); expect(out.prefixCacheHitRate).toEqual([]); }); + + it('extracts TensorRT-LLM KV utilization and prefix cache hit rate', () => { + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + { + timeslices: [ + { start_ns: 0, avg: 0.2 }, + { start_ns: 1, avg: 0.6 }, + ], + }, + ], + }, + trtllm_prompt_cached_tokens_total: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 70 }, + { start_ns: 1, rate: 20 }, + ], + }, + ], + }, + trtllm_prompt_tokens_total: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 100 }, + { start_ns: 1, rate: 50 }, + ], + }, + ], + }, + }, + }); + + const out = extractServerMetricSamples(json); + expect(out.kvCacheUtil).toEqual([0.2, 0.6]); + expect(out.prefixCacheHitRate).toEqual([0.7, 0.4]); + }); }); /** The write-back payload as bound to the UPDATE (a partial aggregate_stats). */ diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 9511d2484..b9d1a3e96 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -157,6 +157,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); const kvCacheUtil = [...aggregateSeriesByStart(kvSeriesAll, 'avg', 'avg').values()]; @@ -167,6 +168,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:prefix_cache_hits', 'vllm:gpu_prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens_total', ); const queriesAll = pickFirstNonEmpty( metrics, @@ -174,6 +176,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:gpu_prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsByT = aggregateSeriesByStart(hitsAll, 'rate', 'sum'); const qByT = aggregateSeriesByStart(queriesAll, 'rate', 'sum'); @@ -182,6 +185,14 @@ export function extractServerMetricSamples(json: string): { const q = qByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push(h / q); } + if (prefixCacheHitRate.length === 0) { + const directRate = aggregateSeriesByStart( + metrics['trtllm_kv_cache_hit_rate']?.series ?? [], + 'avg', + 'avg', + ); + prefixCacheHitRate.push(...directRate.values()); + } return { kvCacheUtil, prefixCacheHitRate }; } @@ -200,6 +211,11 @@ const TARGET_METRIC_KEYS = new Set([ 'sglang:token_usage', 'sglang:cached_tokens', 'sglang:prompt_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens_total', ]); /** From 1e4878604ac641741c06d8d8ea40c2608483cca4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 17 Aug 2026 10:28:20 -0500 Subject: [PATCH 02/10] fix(agentx): handle AIPerf-normalized TRT metric names --- packages/app/src/lib/api-route-catalog.ts | 2 +- .../db/src/etl/compute-aggregate-stats.ts | 2 + .../db/src/etl/compute-chart-series.test.ts | 6 +- packages/db/src/etl/compute-chart-series.ts | 75 +++++++++++-------- .../db/src/queries/agentic-aggregates.test.ts | 4 +- packages/db/src/queries/agentic-aggregates.ts | 4 + 6 files changed, 56 insertions(+), 37 deletions(-) diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f93535dca..667052474 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -711,7 +711,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/agentic-aggregates.ts', - sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18', + sourceSha256: 'b8b72a37ca7a67a1f234036fcbd9e3edacbd7031e0a68fb793944fc4de7030da', reviewArea: { en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.', zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。', diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 54a896441..85cd7c1ac 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -195,7 +195,9 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([ 'vllm:gpu_prefix_cache_queries', 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ]); diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 4b93ae0ea..97766cc7f 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -736,19 +736,19 @@ describe('computeChartSeries', () => { trtllm_kv_cache_host_utilization: { series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], }, - trtllm_prompt_tokens_total: { + trtllm_prompt_tokens: { series: [ buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'), buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'), ], }, - trtllm_prompt_cached_tokens_total: { + trtllm_prompt_cached_tokens: { series: [ buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'), buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'), ], }, - trtllm_generation_tokens_total: { + trtllm_generation_tokens: { series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')], }, trtllm_num_requests_running: { diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 39d4aac61..416c95329 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -250,8 +250,11 @@ export const CHART_METRIC_KEYS = new Set([ 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_host_utilization', 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', + 'trtllm_generation_tokens', 'trtllm_generation_tokens_total', 'trtllm_num_requests_running', 'trtllm_num_requests_waiting', @@ -1004,12 +1007,14 @@ function buildSeriesFromMetrics( const hitsSeries = pickSeries( 'vllm:prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); const qsSeries = pickSeries( 'vllm:prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ); const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS); @@ -1021,11 +1026,11 @@ function buildSeriesFromMetrics( if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q }); } if (prefixCacheHitRate.length === 0) { - for (const [t, value] of sortedEntries( - aggregateByStart(metrics['trtllm_kv_cache_hit_rate']?.series, 'avg', 'avg'), - )) { - prefixCacheHitRate.push({ t: tOf(t), value }); - } + prefixCacheHitRate.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_hit_rate']?.series, tOf), + ), + ); } // Queue depth: sum running + waiting across engines per timeslice. @@ -1060,11 +1065,13 @@ function buildSeriesFromMetrics( const prefillTps = counterRate( 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ); const decodeTps = counterRate( 'vllm:generation_tokens', 'sglang:generation_tokens', + 'trtllm_generation_tokens', 'trtllm_generation_tokens_total', ); // Tokens served from prefix cache per scrape. Lets the frontend derive @@ -1072,6 +1079,7 @@ function buildSeriesFromMetrics( const prefixCacheHitsTps = counterRate( 'vllm:prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); @@ -1094,11 +1102,11 @@ function buildSeriesFromMetrics( } } if (hostKvCacheUsage.length === 0) { - for (const [t, value] of sortedEntries( - aggregateByStart(metrics['trtllm_kv_cache_host_utilization']?.series, 'avg', 'avg'), - )) { - hostKvCacheUsage.push({ t: tOf(t), value }); - } + hostKvCacheUsage.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_host_utilization']?.series, tOf), + ), + ); } // Per-source prompt tokens — sum across engines per source label. @@ -1159,27 +1167,6 @@ function buildSeriesFromMetrics( addSeriesRates(label, series); } } - if (promptBySrcByT.size === 0) { - const promptByT = aggregateByStart( - metrics['trtllm_prompt_tokens_total']?.series, - 'rate', - 'sum', - ); - const cachedByT = aggregateByStart( - metrics['trtllm_prompt_cached_tokens_total']?.series, - 'rate', - 'sum', - ); - const cachedSeries: RawSeries = { timeslices: [] }; - const computedSeries: RawSeries = { timeslices: [] }; - for (const [t, prompt] of promptByT) { - const cached = Math.max(0, cachedByT.get(t) ?? 0); - cachedSeries.timeslices!.push({ start_ns: t, rate: cached }); - computedSeries.timeslices!.push({ start_ns: t, rate: Math.max(0, prompt - cached) }); - } - addSeriesRates('cache hit (HBM)', cachedSeries); - addSeriesRates('compute (miss)', computedSeries); - } const promptTokensBySource: Record = {}; for (const [source, seriesForSource] of promptBySrc) { // Idle ticks are dropped rather than emitted as zeros: this feeds a @@ -1187,6 +1174,32 @@ function buildSeriesFromMetrics( const arr = summedSeries(seriesForSource, tOf, 'rate', tickS).filter((p) => p.value > 0); if (arr.length > 0) promptTokensBySource[source] = arr; } + if (Object.keys(promptTokensBySource).length === 0) { + const promptPoints = summedSeries( + pickSeries('trtllm_prompt_tokens', 'trtllm_prompt_tokens_total'), + tOf, + 'rate', + tickS, + ); + const cachedByT = byT( + summedSeries( + pickSeries('trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total'), + tOf, + 'rate', + tickS, + ), + ); + const cached: TimeSeriesPoint[] = []; + const computed: TimeSeriesPoint[] = []; + for (const { t, value: prompt } of promptPoints) { + const cachedValue = Math.max(0, cachedByT.get(t) ?? 0); + if (cachedValue > 0) cached.push({ t, value: cachedValue }); + const computedValue = Math.max(0, prompt - cachedValue); + if (computedValue > 0) computed.push({ t, value: computedValue }); + } + if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached; + if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed; + } const metricSources: MetricSourceSeries[] = []; const adapter = selectServerMetricsAdapter(context); diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 7882c0741..945b515a4 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -143,7 +143,7 @@ describe('extractServerMetricSamples', () => { }, ], }, - trtllm_prompt_cached_tokens_total: { + trtllm_prompt_cached_tokens: { series: [ { timeslices: [ @@ -153,7 +153,7 @@ describe('extractServerMetricSamples', () => { }, ], }, - trtllm_prompt_tokens_total: { + trtllm_prompt_tokens: { series: [ { timeslices: [ diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index b9d1a3e96..aecb32611 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -168,6 +168,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:prefix_cache_hits', 'vllm:gpu_prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); const queriesAll = pickFirstNonEmpty( @@ -176,6 +177,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:gpu_prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ); const hitsByT = aggregateSeriesByStart(hitsAll, 'rate', 'sum'); @@ -214,7 +216,9 @@ const TARGET_METRIC_KEYS = new Set([ // TensorRT-LLM 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ]); From 10cdc42f057262f31ccb438fbcbc440f7d698964 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 28 Aug 2026 10:22:16 -0500 Subject: [PATCH 03/10] fix(agentx): derive TRT prompt throughput from prefill histogram --- .../db/src/etl/compute-chart-series.test.ts | 11 +-- packages/db/src/etl/compute-chart-series.ts | 82 ++++++++++++------- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 97766cc7f..9edb72f99 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -150,12 +150,13 @@ function buildTrtllmSeries( endpoint_url: string, dynamo_component: 'prefill' | 'backend', value: number, - field: 'rate' | 'avg', + field: 'rate' | 'avg' | 'sum', + durationNs = 1e9, ) { return { endpoint_url, labels: { dynamo_component, worker_id: `${dynamo_component}-worker` }, - timeslices: [{ start_ns: 0, end_ns: 1e9, [field]: value }], + timeslices: [{ start_ns: 0, end_ns: durationNs, [field]: value }], }; } @@ -736,10 +737,10 @@ describe('computeChartSeries', () => { trtllm_kv_cache_host_utilization: { series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], }, - trtllm_prompt_tokens: { + trtllm_prefill_batch_tokens: { series: [ - buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'), - buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'), + buildTrtllmSeries(prefillUrl, 'prefill', 30, 'sum', 0.5e9), + buildTrtllmSeries(decodeUrl, 'backend', 60, 'sum', 0.5e9), ], }, trtllm_prompt_cached_tokens: { diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 416c95329..c6a88f3f7 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -122,8 +122,13 @@ import { * * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV * metrics, including per-prefill/decode source series for Dynamo disaggregation. + * + * v17: derive TensorRT-LLM prompt throughput from its actual metric shape: + * cached-token counter rate plus the prefill-batch-token histogram sum per + * timeslice. TRT-LLM does not currently expose the prompt-token counter that + * v16 expected. */ -export const CHART_SERIES_VERSION = 16; +export const CHART_SERIES_VERSION = 17; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -206,8 +211,11 @@ interface RawSlice { end_ns?: number; avg?: number; rate?: number; + sum?: number; } +type RawSliceField = 'avg' | 'rate' | 'sumRate'; + interface RawSeries { endpoint_url?: string; labels?: Record; @@ -254,6 +262,7 @@ export const CHART_METRIC_KEYS = new Set([ 'trtllm_prompt_cached_tokens_total', 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', + 'trtllm_prefill_batch_tokens', 'trtllm_generation_tokens', 'trtllm_generation_tokens_total', 'trtllm_num_requests_running', @@ -487,7 +496,7 @@ const MIRROR_RATE_RELATIVE_TOLERANCE = 0.05; * request counts) whose mirrors agree almost exactly. Rates need a relative * test because their magnitude is unbounded. */ -function looksMirrored(means: readonly number[], field: 'avg' | 'rate'): boolean { +function looksMirrored(means: readonly number[], field: RawSliceField): boolean { const spread = Math.max(...means) - Math.min(...means); if (field === 'avg') return spread <= MIRROR_MEAN_TOLERANCE; const scale = Math.max(...means.map((m) => Math.abs(m))); @@ -569,7 +578,7 @@ function spanOf(scrapes: ScrapeMap): number { function resolveComponents( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate' = 'avg', + field: RawSliceField = 'avg', ): ResolvedEngine[] { const groups = new Map(); for (const s of series ?? []) { @@ -587,7 +596,16 @@ function resolveComponents( } for (const ts of s.timeslices ?? []) { if (typeof ts.start_ns !== 'number' || !Number.isFinite(ts.start_ns)) continue; - const value = ts[field]; + const durationS = + typeof ts.start_ns === 'number' && typeof ts.end_ns === 'number' + ? (ts.end_ns - ts.start_ns) / 1e9 + : 0; + const value = + field === 'sumRate' + ? typeof ts.sum === 'number' && durationS > 0 + ? ts.sum / durationS + : undefined + : ts[field]; if (typeof value !== 'number' || !Number.isFinite(value)) continue; const at = scrapes.get(ts.start_ns); if (at) { @@ -929,7 +947,7 @@ function sumOntoGrid( function summedSeries( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate', + field: RawSliceField, tickS: number | null, ): TimeSeriesPoint[] { const components = resolveComponents(series, tOf, field).map((c) => c.points); @@ -1062,7 +1080,7 @@ function buildSeriesFromMetrics( // work. const counterRate = (...names: string[]): TimeSeriesPoint[] => summedSeries(pickSeries(...names), tOf, 'rate', tickS); - const prefillTps = counterRate( + const promptCounterTps = counterRate( 'vllm:prompt_tokens', 'sglang:prompt_tokens', 'trtllm_prompt_tokens', @@ -1082,6 +1100,25 @@ function buildSeriesFromMetrics( 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); + const trtllmComputedPromptTps = summedSeries( + metrics['trtllm_prefill_batch_tokens']?.series, + tOf, + 'sumRate', + tickS, + ); + const prefillTps = + promptCounterTps.length > 0 + ? promptCounterTps + : sumOntoGrid([trtllmComputedPromptTps, prefixCacheHitsTps], tickS); + if (prefixCacheHitRate.length === 0 && (!qsSeries || qsSeries.length === 0)) { + const prefillByT = byT(prefillTps); + for (const { t, value: cached } of prefixCacheHitsTps) { + const prompt = prefillByT.get(t); + if (prompt !== undefined && prompt > 0) { + prefixCacheHitRate.push({ t, value: cached / prompt }); + } + } + } // SGLang hicache: host-pool KV cache utilization as used/total per // timeslice. Both metrics are gauges in absolute tokens. Total stays @@ -1175,28 +1212,17 @@ function buildSeriesFromMetrics( if (arr.length > 0) promptTokensBySource[source] = arr; } if (Object.keys(promptTokensBySource).length === 0) { - const promptPoints = summedSeries( - pickSeries('trtllm_prompt_tokens', 'trtllm_prompt_tokens_total'), - tOf, - 'rate', - tickS, - ); - const cachedByT = byT( - summedSeries( - pickSeries('trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total'), - tOf, - 'rate', - tickS, - ), - ); - const cached: TimeSeriesPoint[] = []; - const computed: TimeSeriesPoint[] = []; - for (const { t, value: prompt } of promptPoints) { - const cachedValue = Math.max(0, cachedByT.get(t) ?? 0); - if (cachedValue > 0) cached.push({ t, value: cachedValue }); - const computedValue = Math.max(0, prompt - cachedValue); - if (computedValue > 0) computed.push({ t, value: computedValue }); - } + const cached = prefixCacheHitsTps.filter((point) => point.value > 0); + const cachedByT = byT(prefixCacheHitsTps); + const computed = + trtllmComputedPromptTps.length > 0 + ? trtllmComputedPromptTps.filter((point) => point.value > 0) + : promptCounterTps + .map(({ t, value: prompt }) => ({ + t, + value: Math.max(0, prompt - (cachedByT.get(t) ?? 0)), + })) + .filter((point) => point.value > 0); if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached; if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed; } From 80f9aec76473471c904499344deeb8f9d293c13b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 31 Aug 2026 11:45:05 -0500 Subject: [PATCH 04/10] Fix staged metric-series refreshes --- .github/workflows/stage-results.yml | 28 ++++++++++++++++++++++-- packages/db/src/backfill-chart-series.ts | 21 ++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index 9f91a765c..b38576f53 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -265,10 +265,34 @@ jobs: INFX_MAIN_PAT: ${{ secrets.INFX_MAIN_PAT }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + refresh-chart-series: + name: Refresh staged chart series + needs: [validate, ingest] + runs-on: blacksmith-16vcpu-ubuntu-2404 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + - name: Recompute stale chart series for staged run + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_STAGING_WRITE_URL }} + RUN_ID: ${{ needs.validate.outputs.run-id }} + run: bun run --cwd packages/db db:backfill-chart-series --run-id "$RUN_ID" --yes + report: name: Report staging result if: ${{ always() && needs.validate.result == 'success' }} - needs: [validate, sync-staging-branch, prepare-staging-database, ingest] + needs: [validate, sync-staging-branch, prepare-staging-database, ingest, refresh-chart-series] runs-on: ubuntu-latest permissions: {} steps: @@ -280,7 +304,7 @@ jobs: PR_NUMBER: ${{ needs.validate.outputs.pr-number }} REQUESTED_BY: ${{ needs.validate.outputs.requested-by }} COMMENT_ID: ${{ needs.validate.outputs.comment-id }} - STAGE_SUCCEEDED: ${{ needs['sync-staging-branch'].result == 'success' && needs['prepare-staging-database'].result == 'success' && needs.ingest.result == 'success' }} + STAGE_SUCCEEDED: ${{ needs['sync-staging-branch'].result == 'success' && needs['prepare-staging-database'].result == 'success' && needs.ingest.result == 'success' && needs['refresh-chart-series'].result == 'success' }} with: github-token: ${{ secrets.PAT }} script: | diff --git a/packages/db/src/backfill-chart-series.ts b/packages/db/src/backfill-chart-series.ts index 800499534..10f58503f 100644 --- a/packages/db/src/backfill-chart-series.ts +++ b/packages/db/src/backfill-chart-series.ts @@ -21,6 +21,7 @@ * [--force] recompute every row, even if version already matches * [--shard-count N] split work across N independent processes * [--shard-index N] zero-based shard handled by this process + * [--run-id N] only process trace rows linked to one GitHub workflow run * [--yes] skip the confirmation prompt */ @@ -35,6 +36,12 @@ import { } from './lib/backfill-runner.js'; const flags = parseLimitForceFlags(); +const runIdIndex = process.argv.indexOf('--run-id'); +const runIdRaw = runIdIndex === -1 ? undefined : process.argv[runIdIndex + 1]; +if (runIdIndex !== -1 && (!runIdRaw || !/^\d+$/u.test(runIdRaw))) { + throw new Error('--run-id must be followed by a numeric GitHub workflow run ID'); +} +const githubRunId = runIdRaw ? Number(runIdRaw) : undefined; const sql = createAdminSql({ noSsl: hasNoSslFlag(), @@ -48,6 +55,7 @@ async function main(): Promise { console.log(` force = ${flags.force}`); console.log(` limit = ${flags.limit ?? 'none'}`); console.log(` shard = ${flags.shardIndex + 1}/${flags.shardCount}`); + console.log(` run_id = ${githubRunId ?? 'all'}`); await runCandidateIdBackfill( async () => { @@ -56,12 +64,24 @@ async function main(): Promise { // null and the API serves them via the slow path (which also returns // null because there's no blob to parse — so the page falls into the // "no stored trace_replay blob" branch). + const runFilter = githubRunId + ? sql` + and exists ( + select 1 + from benchmark_results br + join latest_workflow_runs wr on wr.id = br.workflow_run_id + where br.trace_replay_id = agentic_trace_replay.id + and wr.github_run_id = ${githubRunId} + ) + ` + : sql``; const candidates = flags.force ? await sql<{ id: number }[]>` select id from agentic_trace_replay where server_metrics_json_gz is not null and mod(id, ${flags.shardCount}) = ${flags.shardIndex} + ${runFilter} -- Restore the newest, most actively viewed runs first. The backfill is -- idempotent, so an interrupted pass resumes with only stale rows. order by id desc @@ -76,6 +96,7 @@ async function main(): Promise { chart_series is null or coalesce((chart_series->>'version')::int, -1) <> ${CHART_SERIES_VERSION} ) + ${runFilter} -- Restore the newest, most actively viewed runs first. The backfill is -- idempotent, so an interrupted pass resumes with only stale rows. order by id desc From d44429dde192a474b2dcf5a1fae8918666b41564 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 31 Aug 2026 19:58:10 -0500 Subject: [PATCH 05/10] Label TRTLLM cache hits as combined --- .../agentic-point/point-summary.test.ts | 17 +++++++++++++++++ .../inference/agentic-point/point-summary.tsx | 11 +++++++++-- .../inference/utils/tooltip-utils.test.ts | 17 +++++++++++++++++ .../components/inference/utils/tooltipUtils.ts | 9 +++++++-- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/app/src/components/inference/agentic-point/point-summary.test.ts b/packages/app/src/components/inference/agentic-point/point-summary.test.ts index 7fef6f8d0..4dbfd8288 100644 --- a/packages/app/src/components/inference/agentic-point/point-summary.test.ts +++ b/packages/app/src/components/inference/agentic-point/point-summary.test.ts @@ -68,6 +68,23 @@ describe('PointSummary', () => { expect(html).toContain('42.00%'); }); + it('labels TRTLLM offload cache hits as combined instead of showing an empty CPU rate', () => { + const html = renderToStaticMarkup( + createElement(PointSummary, { + meta: meta({ + framework: 'dynamo-trt', + kv_offloading: 'dram', + server_gpu_cache_hit_rate: 0.978, + server_cpu_cache_hit_rate: null, + }), + }), + ); + + expect(html).toContain('Combined chip + CPU cache hit'); + expect(html).toContain('97.80%'); + expect(html).not.toContain('>CPU cache hit<'); + }); + it('shows runtime component names and independently reported versions', () => { const html = renderToStaticMarkup( createElement(PointSummary, { diff --git a/packages/app/src/components/inference/agentic-point/point-summary.tsx b/packages/app/src/components/inference/agentic-point/point-summary.tsx index 1bd6b3a0d..99c10cf2b 100644 --- a/packages/app/src/components/inference/agentic-point/point-summary.tsx +++ b/packages/app/src/components/inference/agentic-point/point-summary.tsx @@ -3,6 +3,7 @@ import type { ReactNode } from 'react'; import type { PointMeta } from '@/hooks/api/use-trace-server-metrics'; +import { frameworkFamily } from '@/lib/framework-family'; import type { Locale } from '@/lib/i18n'; import { isKvOffloadEnabled } from '@/lib/kv-offload'; import { useLocale } from '@/lib/use-locale'; @@ -24,6 +25,7 @@ export const POINT_SUMMARY_STRINGS = { concurrency: 'Concurrency', gpuCacheHit: 'Chip cache hit', cpuCacheHit: 'CPU cache hit', + combinedCacheHit: 'Combined chip + CPU cache hit', enabledLegacy: 'Enabled (legacy data)', disabledLegacy: 'Disabled (legacy data)', none: 'None', @@ -40,6 +42,7 @@ export const POINT_SUMMARY_STRINGS = { concurrency: '并发数', gpuCacheHit: '芯片 cache 命中率', cpuCacheHit: 'CPU cache 命中率', + combinedCacheHit: '芯片 + CPU 综合 cache 命中率', enabledLegacy: '已启用（旧版数据）', disabledLegacy: '已禁用（旧版数据）', none: '无', @@ -71,6 +74,7 @@ export function PointSummary({ meta }: { meta: PointMeta }) { const locale = useLocale(); const t = POINT_SUMMARY_STRINGS[locale]; const showCpuCacheHit = isKvOffloadEnabled(meta); + const showCombinedCacheHit = showCpuCacheHit && frameworkFamily(meta.framework) === 'trt'; const offloadBackend = versionedComponentLabel( meta.kv_offload_backend, meta.kv_offload_backend_version, @@ -104,8 +108,11 @@ export function PointSummary({ meta }: { meta: PointMeta }) { {transferEngine && } {router && } - - {showCpuCacheHit && ( + + {showCpuCacheHit && !showCombinedCacheHit && ( )} {meta.isl !== null && } diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index 743942710..363b84891 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -500,6 +500,23 @@ describe('generateTooltipContent', () => { expect(enabled).toContain('CPU Cache Hit Rate: 42.0%'); }); + it('labels TRTLLM offload cache hits as a combined chip and CPU rate', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ + framework: 'dynamo-trt', + kv_offloading: 'dram', + server_gpu_cache_hit_rate: 0.978, + server_cpu_cache_hit_rate: undefined, + }), + }), + ); + + expect(html).toContain('Combined Chip + CPU Cache Hit Rate: 97.8%'); + expect(html).not.toContain('CPU Cache Hit Rate:'); + expect(html).not.toContain('Chip Cache Hit Rate:'); + }); + it('uses Chinese labels for new cache metadata on /zh surfaces', () => { const html = generateTooltipContent( tooltipConfig({ diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 731ec54b8..11d51f81b 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -2,6 +2,7 @@ import { formatNumber, getDisplayLabel } from '@/lib/utils'; import { specMethodDisplayLabel } from '@/lib/compare-variant-slug'; import { agenticDetailHref } from '@/lib/agentic-detail-link'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; +import { frameworkFamily } from '@/lib/framework-family'; import type { Locale } from '@/lib/i18n'; import { isKvOffloadEnabled } from '@/lib/kv-offload'; @@ -167,6 +168,7 @@ const CACHE_STRINGS = { router: 'Router', gpuHitRate: 'Chip Cache Hit Rate', cpuHitRate: 'CPU Cache Hit Rate', + combinedHitRate: 'Combined Chip + CPU Cache Hit Rate', theoreticalHitRate: 'Theoretical Cache Hit Rate', legacyEnabled: 'Enabled (legacy data)', legacyDisabled: 'Disabled (legacy data)', @@ -178,6 +180,7 @@ const CACHE_STRINGS = { router: '路由器', gpuHitRate: '芯片 Cache 命中率', cpuHitRate: 'CPU Cache 命中率', + combinedHitRate: '芯片 + CPU 综合 Cache 命中率', theoreticalHitRate: '理论 Cache 命中率', legacyEnabled: '已启用（旧版数据）', legacyDisabled: '已禁用（旧版数据）', @@ -216,8 +219,10 @@ const generateCacheMetadataHTML = (d: InferenceData, locale: Locale): string => const gpuHit = formatPct(d.server_gpu_cache_hit_rate); const cpuHit = formatPct(d.server_cpu_cache_hit_rate); const theoreticalHit = formatPct(d.theoretical_cache_hit_rate); - if (gpuHit) parts.push(tooltipLine(t.gpuHitRate, gpuHit)); - if (cpuHit && isKvOffloadEnabled(d)) parts.push(tooltipLine(t.cpuHitRate, cpuHit)); + const offloadEnabled = isKvOffloadEnabled(d); + const combinedHit = offloadEnabled && frameworkFamily(d.framework) === 'trt'; + if (gpuHit) parts.push(tooltipLine(combinedHit ? t.combinedHitRate : t.gpuHitRate, gpuHit)); + if (cpuHit && offloadEnabled && !combinedHit) parts.push(tooltipLine(t.cpuHitRate, cpuHit)); if (theoreticalHit) parts.push(tooltipLine(t.theoreticalHitRate, theoreticalHit)); return parts.join(''); }; From 2fdb175697eb762bcf13765fdf25c989566177e0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 31 Aug 2026 20:27:30 -0500 Subject: [PATCH 06/10] fix(agentic): wrap point-detail chart legends instead of overlapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-rolled point-detail charts laid their legend out on an equal-width grid (`innerW / itemCount`) with no regard for how long the labels actually are. At the inline 720-unit render the KV-cache chart packs eleven per-engine series into ~58 units each, so "prefill (500e)", "decode (5021)" and the rest overprinted one another into an unreadable smear. Add a shared legend layout helper that estimates each label's advance width from its characters (fullwidth CJK counted at ~1em, Latin by character class, rounded up so we wrap early rather than collide) and packs items greedily into as many rows as they need. Charts grow their viewBox height and bottom padding by the same amount, so the plot area keeps its exact geometry and the extra rows extend the SVG downward — a legend that already fit on one row renders exactly where it did before. Applied to all three charts that shared the pattern: TimeSeriesChart, StackedAreaChart, and the distribution percentile chips. Legend items are now packed left rather than spread across the plot width. Verified at inline (720), narrow, and expanded (1300) sizes with the eleven real KV-cache labels, with CJK labels, and against real trace data for point 439515 inline and in the expanded dialog. 中文：修复 agentic 明细页图表图例重叠问题。 这些手写 SVG 图表此前按 `innerW / 图例项数` 等宽分配图例位置，完全没有 考虑标签实际长度。在 720 单位的内联渲染下，KV cache 图表要放下 11 条 per-engine 曲线，每项只有约 58 单位，导致 "prefill (500e)"、"decode (5021)" 等标签互相叠印，完全无法辨认。 新增共享的图例排版辅助模块：按字符估算每个标签的宽度（CJK 全角字符按约 1em 计算，拉丁字符按字符类别估算，并统一向上取整，宁可提前换行也不重叠）， 再贪心地将图例项分配到所需的行数。图表将 viewBox 高度与底部内边距同步增加 相同数值，因此绘图区几何尺寸完全不变，多出的行向下延伸——原本单行就能放下的 图例，渲染位置与改动前完全一致。 该修复已应用到共用此排版逻辑的全部三个图表：TimeSeriesChart、 StackedAreaChart 以及分布图的分位数图例。图例项现在改为左对齐紧凑排列， 不再横向铺满绘图区宽度。 已在内联（720）、窄容器、展开（1300）三种尺寸下，使用真实的 11 条 KV cache 标签及 CJK 标签验证，并针对 439515 号数据点的真实 trace 数据验证了内联渲染 与展开弹窗。 Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic-point/chart-legend.test.ts | 135 ++++++++++++++++++ .../inference/agentic-point/chart-legend.ts | 122 ++++++++++++++++ .../inference/agentic-point/chart-shared.tsx | 95 ++++++++++++ .../inference/agentic-point/distribution.tsx | 74 +++++----- .../agentic-point/time-series-chart.tsx | 128 ++++++++++------- 5 files changed, 466 insertions(+), 88 deletions(-) create mode 100644 packages/app/src/components/inference/agentic-point/chart-legend.test.ts create mode 100644 packages/app/src/components/inference/agentic-point/chart-legend.ts diff --git a/packages/app/src/components/inference/agentic-point/chart-legend.test.ts b/packages/app/src/components/inference/agentic-point/chart-legend.test.ts new file mode 100644 index 000000000..be475d90a --- /dev/null +++ b/packages/app/src/components/inference/agentic-point/chart-legend.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; + +import { + LEGEND_ITEM_GAP, + LEGEND_ROW_HEIGHT, + LEGEND_TEXT_OFFSET, + estimateTextWidth, + layoutChartLegend, +} from './chart-legend'; + +// Inline render of the point-detail charts: 720 viewBox units wide with the +// shared 60/16 left/right axis padding. +const INLINE_INNER_WIDTH = 720 - 60 - 16; + +/** Right edge of the item at `index`, i.e. where its text stops. */ +const itemRight = (labels: readonly string[], index: number, width: number): number => { + const { items } = layoutChartLegend(labels, width); + return items[index]!.x + LEGEND_TEXT_OFFSET + estimateTextWidth(labels[index]!); +}; + +describe('estimateTextWidth', () => { + it('grows with the number of characters', () => { + expect(estimateTextWidth('decode')).toBeGreaterThan(estimateTextWidth('dec')); + }); + + it('counts CJK characters as roughly one em each', () => { + // Four fullwidth characters at font size 11. + expect(estimateTextWidth('芯片缓存')).toBeCloseTo(44, 5); + }); + + it('charges more for wide characters than narrow ones', () => { + expect(estimateTextWidth('mmmm')).toBeGreaterThan(estimateTextWidth('llll')); + }); + + it('returns zero for an empty label', () => { + expect(estimateTextWidth('')).toBe(0); + }); + + it('scales linearly with font size', () => { + expect(estimateTextWidth('decode', 22)).toBeCloseTo(estimateTextWidth('decode', 11) * 2, 5); + }); +}); + +describe('layoutChartLegend', () => { + it('keeps a short legend on one row and reserves no extra height', () => { + const layout = layoutChartLegend(['Input', 'Decode'], INLINE_INNER_WIDTH); + expect(layout.rows).toBe(1); + expect(layout.extraHeight).toBe(0); + expect(layout.items.map((i) => i.row)).toEqual([0, 0]); + }); + + it('lays same-row items out left to right without overlapping', () => { + const labels = ['Input', 'Decode']; + const { items } = layoutChartLegend(labels, INLINE_INNER_WIDTH); + expect(items[0]!.x).toBe(0); + expect(items[1]!.x).toBe(LEGEND_TEXT_OFFSET + estimateTextWidth(labels[0]!) + LEGEND_ITEM_GAP); + expect(items[1]!.x).toBeGreaterThanOrEqual(itemRight(labels, 0, INLINE_INNER_WIDTH)); + }); + + it('wraps the eleven-series KV-cache legend instead of overprinting it', () => { + // The regression from the inline KV-cache chart: eleven per-engine labels + // in ~644 units used to get 58 units each and collide. + const labels = [ + 'prefill (500e)', + 'prefill (501a)', + 'prefill (501e)', + 'decode (5021)', + 'decode (5023)', + 'decode (5025)', + 'decode (5027)', + 'Chip HBM (avg n=50)', + 'DRAM', + 'CPU offload pool (avg n=50)', + 'Avg', + ]; + const layout = layoutChartLegend(labels, INLINE_INNER_WIDTH); + + expect(layout.rows).toBeGreaterThan(1); + expect(layout.extraHeight).toBe((layout.rows - 1) * LEGEND_ROW_HEIGHT); + + for (const [i, label] of labels.entries()) { + const item = layout.items[i]!; + const right = item.x + LEGEND_TEXT_OFFSET + estimateTextWidth(label); + // Every item fits inside the plot width... + expect(right).toBeLessThanOrEqual(INLINE_INNER_WIDTH); + // ...and starts clear of its predecessor on the same row. + const prev = layout.items[i - 1]; + if (prev && prev.row === item.row) { + expect(item.x).toBeGreaterThanOrEqual( + prev.x + LEGEND_TEXT_OFFSET + estimateTextWidth(labels[i - 1]!), + ); + } + } + }); + + it('starts each wrapped row back at the left edge', () => { + const labels = Array.from({ length: 12 }, (_, i) => `series number ${i}`); + const layout = layoutChartLegend(labels, INLINE_INNER_WIDTH); + const firstOfRow = new Map(); + for (const item of layout.items) { + if (!firstOfRow.has(item.row)) firstOfRow.set(item.row, item.x); + } + for (const x of firstOfRow.values()) expect(x).toBe(0); + }); + + it('wraps sooner for Chinese labels, which are wider per character', () => { + const en = Array.from({ length: 6 }, () => 'Chip HBM pool'); + const zh = Array.from({ length: 6 }, () => '芯片 HBM 显存池均值'); + expect(layoutChartLegend(zh, INLINE_INNER_WIDTH).rows).toBeGreaterThanOrEqual( + layoutChartLegend(en, INLINE_INNER_WIDTH).rows, + ); + }); + + it('needs fewer rows at the expanded width than inline', () => { + const labels = Array.from({ length: 11 }, (_, i) => `decode engine ${i}`); + const inline = layoutChartLegend(labels, INLINE_INNER_WIDTH); + const expanded = layoutChartLegend(labels, 1300 - 60 - 16); + expect(expanded.rows).toBeLessThan(inline.rows); + }); + + it('gives a label wider than the row its own row rather than dropping it', () => { + const layout = layoutChartLegend(['a', 'w'.repeat(300), 'b'], 100); + expect(layout.items).toHaveLength(3); + expect(layout.items[1]!.row).toBe(1); + expect(layout.items[1]!.x).toBe(0); + expect(layout.items[2]!.row).toBe(2); + }); + + it('reports a single row for an empty legend', () => { + const layout = layoutChartLegend([], INLINE_INNER_WIDTH); + expect(layout.items).toEqual([]); + expect(layout.rows).toBe(1); + expect(layout.extraHeight).toBe(0); + }); +}); diff --git a/packages/app/src/components/inference/agentic-point/chart-legend.ts b/packages/app/src/components/inference/agentic-point/chart-legend.ts new file mode 100644 index 000000000..5dfc72dfd --- /dev/null +++ b/packages/app/src/components/inference/agentic-point/chart-legend.ts @@ -0,0 +1,122 @@ +/** + * Legend layout for the hand-rolled agentic point-detail charts. + * + * Those charts are plain SVG inside a fixed viewBox that scales to the card + * width, so there is no DOM to measure text against at layout time. The legend + * used to sit on an equal-width grid (`innerW / itemCount`), which collides as + * soon as the labels are longer than their slot — the inline 720-unit render of + * the KV-cache chart packs eleven per-engine series into ~58 units each and the + * labels overprint one another. + * + * Instead we estimate each label's advance width from its characters and pack + * items greedily into as many rows as they need. Estimates are deliberately a + * little generous: over-estimating wraps one item early, under-estimating + * reintroduces the overlap this module exists to prevent. + */ + +/** Font size the chart legends render at. */ +export const LEGEND_FONT_SIZE = 11; + +/** Baseline-to-baseline distance between wrapped legend rows. */ +export const LEGEND_ROW_HEIGHT = 14; + +/** x offset from an item's origin to the start of its color swatch. */ +export const LEGEND_SWATCH_INSET = 2; + +/** Width of the color swatch (line segment or filled rect). */ +export const LEGEND_SWATCH_WIDTH = 12; + +/** x offset from an item's origin to the start of its text. */ +export const LEGEND_TEXT_OFFSET = 18; + +/** Horizontal gap between two legend items on the same row. */ +export const LEGEND_ITEM_GAP = 12; + +/** Distance from the bottom of the viewBox to the last legend row's baseline. */ +export const LEGEND_BASELINE_OFFSET = 8; + +// Fullwidth scripts (CJK ideographs, kana, Hangul, fullwidth forms) advance +// roughly one em per character rather than the ~0.5em of Latin lowercase. +const FULLWIDTH = + /[\u1100-\u115F\u2E80-\u303E\u3041-\u33FF\u3400-\u4DBF\u4E00-\u9FFF\uA000-\uA4CF\uAC00-\uD7A3\uF900-\uFAFF\uFE30-\uFE4F\uFF00-\uFF60\uFFE0-\uFFE6]/u; + +// Character classes, in em units at the legend font size. Sampled against the +// app's sans stack and rounded up. +const NARROW_CHARS = new Set(" .,:;!|'`ijltfr()[]{}/\\-"); +const WIDE_CHARS = new Set('mwMW@%'); +const UPPER_OR_DIGIT = /[A-Z0-9$#]/u; + +const EM_FULLWIDTH = 1; +const EM_NARROW = 0.34; +const EM_WIDE = 0.88; +const EM_UPPER_OR_DIGIT = 0.62; +const EM_DEFAULT = 0.53; + +/** + * Approximate rendered width of `text` in viewBox units. + * + * Not exact — SVG text has no measurable width until it is in the document — + * but consistently at or slightly above the real advance width, which is the + * side to err on for collision avoidance. + */ +export function estimateTextWidth(text: string, fontSize: number = LEGEND_FONT_SIZE): number { + let em = 0; + for (const ch of text) { + if (FULLWIDTH.test(ch)) em += EM_FULLWIDTH; + else if (NARROW_CHARS.has(ch)) em += EM_NARROW; + else if (WIDE_CHARS.has(ch)) em += EM_WIDE; + else if (UPPER_OR_DIGIT.test(ch)) em += EM_UPPER_OR_DIGIT; + else em += EM_DEFAULT; + } + return em * fontSize; +} + +/** Placement of one legend item within the wrapped legend block. */ +export interface LegendItemLayout { + /** x of the item's origin, relative to the left edge of the plot area. */ + x: number; + /** 0-based row the item was packed into; row 0 is the topmost. */ + row: number; +} + +export interface LegendLayout { + items: LegendItemLayout[]; + /** Number of rows the legend occupies. At least 1, even when empty. */ + rows: number; + /** + * Vertical space, in viewBox units, the legend needs beyond the single row + * the chart already reserves in its bottom padding. Charts add this to both + * their height and their bottom padding so the plot area is unchanged and + * the extra rows extend the SVG downward. + */ + extraHeight: number; +} + +/** + * Pack legend labels into rows no wider than `availableWidth`. + * + * A label wider than the whole row still gets its own row rather than being + * dropped — a clipped label is more useful than a missing one. + */ +export function layoutChartLegend( + labels: readonly string[], + availableWidth: number, + fontSize: number = LEGEND_FONT_SIZE, +): LegendLayout { + const items: LegendItemLayout[] = []; + let row = 0; + let cursor = 0; + + for (const label of labels) { + const itemWidth = LEGEND_TEXT_OFFSET + estimateTextWidth(label, fontSize); + if (cursor > 0 && cursor + itemWidth > availableWidth) { + row += 1; + cursor = 0; + } + items.push({ x: cursor, row }); + cursor += itemWidth + LEGEND_ITEM_GAP; + } + + const rows = items.length === 0 ? 1 : row + 1; + return { items, rows, extraHeight: (rows - 1) * LEGEND_ROW_HEIGHT }; +} diff --git a/packages/app/src/components/inference/agentic-point/chart-shared.tsx b/packages/app/src/components/inference/agentic-point/chart-shared.tsx index b8333b4e9..8379327d2 100644 --- a/packages/app/src/components/inference/agentic-point/chart-shared.tsx +++ b/packages/app/src/components/inference/agentic-point/chart-shared.tsx @@ -2,6 +2,16 @@ import { useLocale } from '@/lib/use-locale'; +import { + LEGEND_BASELINE_OFFSET, + LEGEND_FONT_SIZE, + LEGEND_ROW_HEIGHT, + LEGEND_SWATCH_INSET, + LEGEND_SWATCH_WIDTH, + LEGEND_TEXT_OFFSET, + type LegendLayout, +} from './chart-legend'; + /** * Shared presentational constants and helpers for the agentic point-detail * charts (time-series, stacked-area, distribution, aggregate). These charts @@ -58,3 +68,88 @@ export function ChartEmpty({ height = 260, message }: { height?: number; message export function ChartSkeleton() { return
; } + +/** How a legend entry's color is shown: a stroked line or a filled block. */ +export type LegendSwatch = 'line' | 'dashed-line' | 'area'; + +export interface ChartLegendEntry { + label: string; + color: string; + /** Defaults to `'line'`. */ + swatch?: LegendSwatch; + /** Stroke width for the line swatches. Defaults to 2. */ + strokeWidth?: number; + /** Swatch opacity, to match a translucent area fill. Defaults to 1. */ + opacity?: number; +} + +/** + * Wrapped legend for the hand-rolled point-detail charts. + * + * `layout` comes from `layoutChartLegend()` and must have been computed from + * the same entry order. `baselineY` is the baseline of the *last* row (charts + * pass `height - LEGEND_BASELINE_OFFSET`), so a single-row legend lands exactly + * where it did before wrapping existed and extra rows grow downward into the + * space the chart added to its bottom padding. + */ +export function ChartLegend({ + entries, + layout, + left, + baselineY, +}: { + entries: readonly ChartLegendEntry[]; + layout: LegendLayout; + /** x of the plot area's left edge; item x offsets are relative to it. */ + left: number; + baselineY: number; +}) { + const topRowBaseline = baselineY - (layout.rows - 1) * LEGEND_ROW_HEIGHT; + return ( + <> + {entries.map((entry, i) => { + const placed = layout.items[i]; + if (!placed) return null; + const x = left + placed.x; + const y = topRowBaseline + placed.row * LEGEND_ROW_HEIGHT; + const swatch = entry.swatch ?? 'line'; + return ( + + {swatch === 'area' ? ( + + ) : ( + + )} + + {entry.label} + + + ); + })} + + ); +} + +export { LEGEND_BASELINE_OFFSET }; diff --git a/packages/app/src/components/inference/agentic-point/distribution.tsx b/packages/app/src/components/inference/agentic-point/distribution.tsx index f11d9f20b..f62a61e5e 100644 --- a/packages/app/src/components/inference/agentic-point/distribution.tsx +++ b/packages/app/src/components/inference/agentic-point/distribution.tsx @@ -5,7 +5,16 @@ import { useMemo } from 'react'; import { useLocale } from '@/lib/use-locale'; import { ChartHover, type HoverItem } from './chart-hover'; -import { CHART_PAD, ChartEmpty, PERCENTILE_COLORS, fmtCount } from './chart-shared'; +import { + CHART_PAD, + ChartEmpty, + ChartLegend, + LEGEND_BASELINE_OFFSET, + PERCENTILE_COLORS, + fmtCount, + type ChartLegendEntry, +} from './chart-shared'; +import { layoutChartLegend } from './chart-legend'; import { logHistogram, logTicks, positiveValues } from './lognormal'; import { quantile } from './time-series-math'; @@ -69,7 +78,6 @@ export function Distribution({ }) { const t = STRINGS[useLocale()]; const W = width; - const H = height; const computed = useMemo(() => { const positive = positiveValues(values); @@ -83,16 +91,31 @@ export function Distribution({ // Zero-token requests are real but unplottable on a log axis; they are // reported under the chart rather than silently folded into bin one. excluded: values.length - positive.length, - innerW: W - PAD.left - PAD.right, - innerH: H - PAD.top - PAD.bottom, }; - }, [values, W, H]); + }, [values]); if (!computed) { - return ; + return ; } - const { sorted, histogram, excluded, innerW, innerH } = computed; + const { sorted, histogram, excluded } = computed; const { counts, edges, lnMin, lnMax } = histogram; + + // Wrap the legend, then grow the viewBox and bottom padding by the same + // amount so the plot area is untouched (see TimeSeriesChart). + const legendEntries: ChartLegendEntry[] = GUIDES.map(({ label, q, color }) => ({ + label: `${label} ${fmtCount(quantile(sorted, q))}`, + color, + swatch: 'dashed-line', + })); + const innerW = W - PAD.left - PAD.right; + const legend = layoutChartLegend( + legendEntries.map((e) => e.label), + innerW, + ); + const H = height + legend.extraHeight; + const padBottom = PAD.bottom + legend.extraHeight; + const pad = { ...PAD, bottom: padBottom }; + const innerH = H - PAD.top - padBottom; const min = edges[0]!; const max = edges.at(-1)!; const nBins = counts.length; @@ -135,7 +158,7 @@ export function Distribution({ {sorted.length.toLocaleString()} {t.requests} · {t.range} {fmt(min)}–{fmt(max)} {unit} ·{' '} {t.logScale}
- + {/* y-axis gridlines + labels */} {yTickVals.map((v, i) => { const y = yScale(v); @@ -227,7 +250,7 @@ export function Distribution({ })} {t.countAxis} {/* Percentile legend chips */} - {(() => { - const chipY = H - 8; - const chipW = innerW / GUIDES.length; - return GUIDES.map(({ label: ql, q, color }, i) => { - const x = PAD.left + i * chipW; - return ( - - - - {ql} {fmt(quantile(sorted, q))} - - - ); - }); - })()} + {excluded > 0 && (
diff --git a/packages/app/src/components/inference/agentic-point/time-series-chart.tsx b/packages/app/src/components/inference/agentic-point/time-series-chart.tsx index 1e5a4166c..1c4291df3 100644 --- a/packages/app/src/components/inference/agentic-point/time-series-chart.tsx +++ b/packages/app/src/components/inference/agentic-point/time-series-chart.tsx @@ -5,7 +5,16 @@ import { useMemo } from 'react'; import type { TimeSeriesPoint } from '@/hooks/api/use-trace-server-metrics'; import { ChartHover, type HoverItem } from './chart-hover'; -import { CHART_PAD, ChartEmpty, fmtCount, fmtSeconds } from './chart-shared'; +import { + CHART_PAD, + ChartEmpty, + ChartLegend, + LEGEND_BASELINE_OFFSET, + fmtCount, + fmtSeconds, + type ChartLegendEntry, +} from './chart-shared'; +import { layoutChartLegend } from './chart-legend'; import { interpAt, maxTimeSeriesValue, type ChartSeries } from './time-series-math'; import { useLocale } from '@/lib/use-locale'; @@ -53,11 +62,35 @@ export function TimeSeriesChart({ }: TimeSeriesChartProps) { const locale = useLocale(); const W = width; - const H = height; + + // Legend entries, skipping series flagged hideFromHover so per-engine + // underlays don't take a legend slot with no visible line. + const legendEntries = useMemo( + () => + series + .filter((s) => !s.hideFromHover) + .map((s) => ({ label: s.name, color: s.color, strokeWidth: s.strokeWidth ?? 2 })), + [series], + ); + + // Wrap the legend to as many rows as the labels need, then grow the viewBox + // and the bottom padding by the same amount. The plot area keeps its exact + // geometry and the extra rows extend the SVG downward. + const legend = useMemo( + () => + layoutChartLegend( + legendEntries.map((e) => e.label), + W - PAD.left - PAD.right, + ), + [legendEntries, W], + ); + const H = height + legend.extraHeight; + const padBottom = PAD.bottom + legend.extraHeight; + const pad = useMemo(() => ({ ...PAD, bottom: padBottom }), [padBottom]); const layout = useMemo(() => { const innerW = W - PAD.left - PAD.right; - const innerH = H - PAD.top - PAD.bottom; + const innerH = H - PAD.top - padBottom; const xMax = Math.max(durationS, 1); // Fold reference-line values into the auto max so a ceiling above the data // (e.g. KV-cache pool >> working set) still renders inside the plot. @@ -67,7 +100,7 @@ export function TimeSeriesChart({ const xScale = (t: number) => PAD.left + (t / xMax) * innerW; const yScale = (v: number) => PAD.top + (1 - v / yMax) * innerH; return { innerW, innerH, xMax, yMax, xScale, yScale }; - }, [series, durationS, yMaxOpt, refLines, W, H]); + }, [series, durationS, yMaxOpt, refLines, W, H, padBottom]); const { innerW, innerH, xMax, yMax, xScale, yScale } = layout; @@ -99,7 +132,7 @@ export function TimeSeriesChart({ } return ( - + {/* y-axis gridlines + labels */} {yTickVals.map((v, i) => { const y = yScale(v); @@ -225,7 +258,7 @@ export function TimeSeriesChart({ })} {yAxisLabel} )} - {/* Legend — skip series flagged hideFromHover so per-engine - underlays don't clutter the chip row. */} - {(() => { - const visible = series.filter((s) => !s.hideFromHover); - const chipY = H - 8; - const chipW = innerW / Math.max(1, visible.length); - return visible.map((s, i) => { - const x = PAD.left + i * chipW; - return ( - - - - {s.name} - - - ); - }); - })()} + ); } @@ -333,7 +347,6 @@ export function StackedAreaChart({ const locale = useLocale(); const sourceLabels = SOURCE_LABELS[locale]; const W = width; - const H = height; const computed = useMemo(() => { const entries = Object.entries(sourceSeries).filter(([, v]) => v.length > 0); @@ -397,8 +410,24 @@ export function StackedAreaChart({ } const colorFor = (name: string): string => colorByName.get(name) ?? FALLBACK_PALETTE[0]!; + // Wrap the legend, then grow the viewBox and bottom padding by the same + // amount so the plot area is untouched (see TimeSeriesChart). + const legendEntries: ChartLegendEntry[] = stackOrder.map((name) => ({ + label: sourceLabels[name] ?? name, + color: colorFor(name), + swatch: 'area', + opacity: 0.75, + })); + const legend = layoutChartLegend( + legendEntries.map((e) => e.label), + W - PAD.left - PAD.right, + ); + const H = height + legend.extraHeight; + const padBottom = PAD.bottom + legend.extraHeight; + const pad = { ...PAD, bottom: padBottom }; + const innerW = W - PAD.left - PAD.right; - const innerH = H - PAD.top - PAD.bottom; + const innerH = H - PAD.top - padBottom; const xMax = Math.max(durationS, 1); const xScale = (t: number) => PAD.left + (t / xMax) * innerW; const yScale = (v: number) => PAD.top + (1 - v) * innerH; @@ -443,7 +472,7 @@ export function StackedAreaChart({ const yTickVals = [0, 0.25, 0.5, 0.75, 1]; return ( - + {yTickVals.map((v, i) => { const y = yScale(v); return ( @@ -499,7 +528,7 @@ export function StackedAreaChart({ })} {locale === 'zh' ? '预填充 token 占比' : '% of prefill tokens'} - {(() => { - const chipY = H - 8; - const chipW = innerW / Math.max(1, layers.length); - return layers.map((l, i) => { - const x = PAD.left + i * chipW; - return ( - - - - {sourceLabels[l.name] ?? l.name} - - - ); - }); - })()} + ); } From 22337f61834e102f4e43eeb22b19faa328ee1176 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 11:13:39 -0500 Subject: [PATCH 07/10] feat(ingest): target named Neon branches --- .github/workflows/ingest-agentic-results.yml | 62 ++++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index e9e273777..0c06bb7af 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -41,6 +41,16 @@ on: required: false default: production type: string + neon-branch: + description: Named Neon branch when database-target is neon-branch + required: false + default: '' + type: string + preview-url: + description: Preview deployment URL to invalidate for a named Neon branch + required: false + default: '' + type: string secrets: DATABASE_WRITE_URL: description: Production database write connection for direct ingests @@ -60,6 +70,12 @@ on: INFX_MAIN_PAT: description: Token used to download InferenceX artifacts required: true + NEON_API_KEY: + description: Neon API key used to resolve named child branches + required: false + NEON_PROJECT_ID: + description: Neon project containing named child branches + required: false SLACK_WEBHOOK_URL: description: Optional failure and unmapped-entity notifications required: false @@ -83,6 +99,17 @@ on: - production - dev - staging + - neon-branch + neon-branch: + description: Named Neon branch when database-target is neon-branch + required: false + default: '' + type: string + preview-url: + description: Preview deployment URL to invalidate for a named Neon branch + required: false + default: '' + type: string permissions: {} @@ -148,6 +175,10 @@ jobs: PROTECTION_BYPASS_SECRET_STAGING: ${{ secrets.VERCEL_STAGING_BYPASS_SECRET }} STAGING_SITE_URL: ${{ vars.STAGING_SITE_URL || 'https://inferencemax-app-git-staging-semianalysisai.vercel.app' }} + NEON_API_KEY: ${{ secrets.NEON_API_KEY }} + NEON_PROJECT_ID: ${{ secrets.NEON_PROJECT_ID }} + NEON_BRANCH_NAME: ${{ inputs.neon-branch }} + PREVIEW_SITE_URL: ${{ inputs.preview-url }} run: | case "$REQUESTED_DATABASE_TARGET" in production) @@ -165,6 +196,29 @@ jobs: cache_invalidate_url="${STAGING_SITE_URL%/}/api/v1/invalidate" protection_bypass_secret="$PROTECTION_BYPASS_SECRET_STAGING" ;; + neon-branch) + if [ -z "$NEON_API_KEY" ] || [ -z "$NEON_PROJECT_ID" ] || [ -z "$NEON_BRANCH_NAME" ] || [ -z "$PREVIEW_SITE_URL" ]; then + echo "::error::NEON_API_KEY, NEON_PROJECT_ID, neon-branch, and preview-url are required" + exit 1 + fi + branches=$(curl --retry 3 --retry-all-errors -sSf \ + -H "Authorization: Bearer $NEON_API_KEY" \ + "https://console.neon.tech/api/v2/projects/$NEON_PROJECT_ID/branches?limit=100") + branch_id=$(jq -r --arg name "$NEON_BRANCH_NAME" \ + '.branches[] | select(.name == $name) | .id' <<<"$branches" | head -n 1) + if [ -z "$branch_id" ] || [ "$branch_id" = "null" ]; then + echo "::error::Neon branch not found: $NEON_BRANCH_NAME" + exit 1 + fi + database_write_url=$(npx --yes neonctl connection-string \ + --api-key "$NEON_API_KEY" \ + --project-id "$NEON_PROJECT_ID" \ + --branch-id "$branch_id" \ + --database-name neondb \ + --role-name neondb_owner) + cache_invalidate_url="${PREVIEW_SITE_URL%/}/api/v1/invalidate" + protection_bypass_secret="$PROTECTION_BYPASS_SECRET_STAGING" + ;; *) echo "::error::Unsupported database-target: $REQUESTED_DATABASE_TARGET" exit 1 @@ -175,12 +229,12 @@ jobs: echo "::error::Database secret is empty for target: $REQUESTED_DATABASE_TARGET" exit 1 fi - if [ "$REQUESTED_DATABASE_TARGET" != "staging" ] && [ -z "$cache_invalidate_secret" ]; then + if [ "$REQUESTED_DATABASE_TARGET" != "staging" ] && [ "$REQUESTED_DATABASE_TARGET" != "neon-branch" ] && [ -z "$cache_invalidate_secret" ]; then echo "::error::Cache invalidation secret is empty for target: $REQUESTED_DATABASE_TARGET" exit 1 fi - if [ "$REQUESTED_DATABASE_TARGET" = "staging" ] && [ -z "$protection_bypass_secret" ]; then - echo "::error::Vercel protection bypass secret is empty for staging" + if { [ "$REQUESTED_DATABASE_TARGET" = "staging" ] || [ "$REQUESTED_DATABASE_TARGET" = "neon-branch" ]; } && [ -z "$protection_bypass_secret" ]; then + echo "::error::Vercel protection bypass secret is empty for preview target" exit 1 fi @@ -234,7 +288,7 @@ jobs: - name: Invalidate Vercel cache run: | - if [ "$INGEST_DATABASE_TARGET" = "staging" ]; then + if [ "$INGEST_DATABASE_TARGET" = "staging" ] || [ "$INGEST_DATABASE_TARGET" = "neon-branch" ]; then curl --retry 3 --retry-delay 2 --retry-connrefused -sSf -X POST "$CACHE_INVALIDATE_URL" \ -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" else From 826084908e0e1ef038c06379b34a0f064ba24296 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 11:22:00 -0500 Subject: [PATCH 08/10] fix(ingest): tolerate preview cache invalidation failures --- .github/workflows/ingest-agentic-results.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index 0c06bb7af..aea065d0b 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -288,9 +288,13 @@ jobs: - name: Invalidate Vercel cache run: | - if [ "$INGEST_DATABASE_TARGET" = "staging" ] || [ "$INGEST_DATABASE_TARGET" = "neon-branch" ]; then + if [ "$INGEST_DATABASE_TARGET" = "staging" ]; then curl --retry 3 --retry-delay 2 --retry-connrefused -sSf -X POST "$CACHE_INVALIDATE_URL" \ -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" + elif [ "$INGEST_DATABASE_TARGET" = "neon-branch" ]; then + curl --retry 3 --retry-delay 2 --retry-connrefused -sSf -X POST "$CACHE_INVALIDATE_URL" \ + -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" || \ + echo "::warning::Preview cache invalidation failed; the database ingest is complete" else curl -sSf -X POST "$CACHE_INVALIDATE_URL" \ -H "Authorization: Bearer $CACHE_INVALIDATE_SECRET" || true From f030da63365f3368472b4809240cc3979c180aa2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 11:52:45 -0500 Subject: [PATCH 09/10] fix(metrics): preserve TRT worker ranks and verify child database targets --- .github/workflows/ingest-agentic-results.yml | 20 +++++-- docs/data-pipeline.md | 5 ++ .../db/src/etl/compute-chart-series.test.ts | 55 +++++++++++++++++++ packages/db/src/etl/compute-chart-series.ts | 29 +++++++--- .../db/src/etl/server-metrics-adapters.ts | 9 ++- 5 files changed, 102 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index aea065d0b..6e23cd53b 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -205,18 +205,29 @@ jobs: -H "Authorization: Bearer $NEON_API_KEY" \ "https://console.neon.tech/api/v2/projects/$NEON_PROJECT_ID/branches?limit=100") branch_id=$(jq -r --arg name "$NEON_BRANCH_NAME" \ - '.branches[] | select(.name == $name) | .id' <<<"$branches" | head -n 1) + '.branches[] | select(.name == $name and .parent_id != null and .default != true) | .id' <<<"$branches" | head -n 1) if [ -z "$branch_id" ] || [ "$branch_id" = "null" ]; then echo "::error::Neon branch not found: $NEON_BRANCH_NAME" exit 1 fi - database_write_url=$(npx --yes neonctl connection-string \ + database_write_url=$(npx --yes neonctl connection-string "$branch_id" \ --api-key "$NEON_API_KEY" \ --project-id "$NEON_PROJECT_ID" \ - --branch-id "$branch_id" \ --database-name neondb \ --role-name neondb_owner) + endpoints=$(curl --retry 3 --retry-all-errors -sSf \ + -H "Authorization: Bearer $NEON_API_KEY" \ + "https://console.neon.tech/api/v2/projects/$NEON_PROJECT_ID/endpoints") + database_host=$(DATABASE_URL_TO_CHECK="$database_write_url" node -e \ + 'console.log(new URL(process.env.DATABASE_URL_TO_CHECK).hostname)') + if ! jq -e --arg branch "$branch_id" --arg host "$database_host" \ + '.endpoints[] | select(.branch_id == $branch and .host == $host)' <<<"$endpoints" >/dev/null; then + echo "::error::Resolved database host does not belong to the requested child branch" + exit 1 + fi + echo "Verified child database: $NEON_BRANCH_NAME ($branch_id), host $database_host" cache_invalidate_url="${PREVIEW_SITE_URL%/}/api/v1/invalidate" + cache_invalidate_secret="$INVALIDATE_SECRET_DEFAULT" protection_bypass_secret="$PROTECTION_BYPASS_SECRET_STAGING" ;; *) @@ -293,7 +304,8 @@ jobs: -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" elif [ "$INGEST_DATABASE_TARGET" = "neon-branch" ]; then curl --retry 3 --retry-delay 2 --retry-connrefused -sSf -X POST "$CACHE_INVALIDATE_URL" \ - -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" || \ + -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" \ + -H "Authorization: Bearer $CACHE_INVALIDATE_SECRET" || \ echo "::warning::Preview cache invalidation failed; the database ingest is complete" else curl -sSf -X POST "$CACHE_INVALIDATE_URL" \ diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 176acc32c..c7087e869 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -172,6 +172,11 @@ AIPerf defines the `server_metrics_export.json` envelope, but labels such as wor Adapters are selected from the benchmark's canonical framework, and per-worker series are only emitted for disaggregated configs with a recognized adapter. Unknown orchestrators and non-disaggregated configs retain their aggregate-only series; roles are never guessed from ports or metric names. The frontend only consumes the canonical source identity and never interprets orchestrator-native labels. +Dynamo/TRT-LLM KV charts prefer `dynamo_component_gpu_cache_usage_percent`, which +retains `dp_rank`, over the rankless native `trtllm_kv_cache_utilization` gauge. +Ranks remain inside each worker's source; native-only endpoints retain their fallback. +The all-endpoints view summarizes workers, while selecting a worker exposes its ranks. + ### Logical Engines vs Raw Series A raw series in the blob is one `(scrape endpoint × phase block × label set)` tuple, which is **not** the same as one engine. The KV-cache chart needs one entry per _logical engine_ — one KV pool — so `compute-chart-series.ts` groups series by their Prometheus label set (`seriesIdentityKey`) rather than emitting one entry per raw series. Three kinds of duplication collapse there: diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 9edb72f99..312853d24 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -786,6 +786,61 @@ describe('computeChartSeries', () => { expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]); expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]); }); + + it('keeps Dynamo TRT ranks within each worker and falls back to native-only endpoints', async () => { + const prefillUrls = ['http://prefill-a:7500/metrics', 'http://prefill-b:7500/metrics']; + const decodeUrl = 'http://decode:7500/metrics'; + const result = await computeChartSeries( + gzipSync( + Buffer.from( + JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + ...prefillUrls.map((url) => buildTrtllmSeries(url, 'prefill', 0.9, 'avg')), + buildTrtllmSeries(decodeUrl, 'backend', 0.8, 'avg'), + ], + }, + dynamo_component_gpu_cache_usage_percent: { + series: prefillUrls.flatMap((endpoint_url) => + [0, 1].map((rank) => ({ + endpoint_url, + labels: { + dynamo_component: 'prefill', + model: 'GLM-5.2-NVFP4', + dp_rank: String(rank), + }, + timeslices: [{ start_ns: 0, end_ns: 1e9, avg: rank === 0 ? 0.2 : 0.6 }], + })), + ), + }, + trtllm_num_requests_running: { + series: prefillUrls.map((url) => buildTrtllmSeries(url, 'prefill', 2, 'avg')), + }, + }, + }), + ), + ), + { framework: 'dynamo-trt', disagg: true }, + ); + + expect(result?.metricSources).toHaveLength(3); + expect(result?.kvCacheUsageByEngine).toHaveLength(5); + expect(result?.kvCacheUsage[0]?.value).toBeCloseTo(0.48); + for (const url of prefillUrls) { + const source = result?.metricSources.find((s) => s.source.endpointUrl === url); + expect(source?.source.dpRank).toBeNull(); + expect(source?.kvCacheUsage).toEqual([{ t: 0, value: 0.4 }]); + expect(source?.kvCacheUsageByEngine).toEqual([ + { engineLabel: '0', points: [{ t: 0, value: 0.2 }] }, + { engineLabel: '1', points: [{ t: 0, value: 0.6 }] }, + ]); + expect(source?.queueDepth).toEqual([{ t: 0, running: 2, waiting: 0, total: 2 }]); + } + expect(result?.metricSources.find((s) => s.source.role === 'decode')?.kvCacheUsage).toEqual([ + { t: 0, value: 0.8 }, + ]); + }); }); // ── Summed series on the canonical grid (v14) ─────────────────────────── diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 01006464d..da321b78b 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -127,8 +127,10 @@ import { * cached-token counter rate plus the prefill-batch-token histogram sum per * timeslice. TRT-LLM does not currently expose the prompt-token counter that * v16 expected. + * + * v18: retain Dynamo's per-DP-rank KV gauges within each TRT-LLM worker. */ -export const CHART_SERIES_VERSION = 17; +export const CHART_SERIES_VERSION = 18; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -255,6 +257,7 @@ export const CHART_METRIC_KEYS = new Set([ 'sglang:hicache_host_used_tokens', 'sglang:hicache_host_total_tokens', // TensorRT-LLM + 'dynamo_component_gpu_cache_usage_percent', 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_host_utilization', 'trtllm_kv_cache_hit_rate', @@ -1004,12 +1007,24 @@ function buildSeriesFromMetrics( // KV cache usage (gauge, 0..1) — average across engines so the value // stays a fraction (each engine has its own KV pool). - const kvSeries = pickSeries( - 'vllm:kv_cache_usage_perc', - 'vllm:gpu_cache_usage_perc', - 'sglang:token_usage', - 'trtllm_kv_cache_utilization', - ); + const nativeTrtKv = metrics['trtllm_kv_cache_utilization']?.series ?? []; + const dynamoTrtKv = + selectServerMetricsAdapter(context).id === 'trtllm' + ? (metrics['dynamo_component_gpu_cache_usage_percent']?.series ?? []) + : []; + const rankedEndpoints = new Set(dynamoTrtKv.map((s) => s.endpoint_url)); + // Dynamo gauges omit worker_id. Preserve endpoint identity so two workers + // with similar utilization and the same DP rank cannot be mistaken for mirrors. + const trtKv = [ + ...dynamoTrtKv.map((s) => ({ + ...s, + labels: { ...s.labels, metric_endpoint: s.endpoint_url ?? '' }, + })), + ...nativeTrtKv.filter((s) => !rankedEndpoints.has(s.endpoint_url)), + ]; + const kvSeries = + pickSeries('vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage') ?? + trtKv; // One entry per logical engine (v13) — mirrored API-server frontends and the // warmup/profiling phase split are collapsed here rather than showing up as // extra "engines". diff --git a/packages/db/src/etl/server-metrics-adapters.ts b/packages/db/src/etl/server-metrics-adapters.ts index 45f3bacc1..4807ce002 100644 --- a/packages/db/src/etl/server-metrics-adapters.ts +++ b/packages/db/src/etl/server-metrics-adapters.ts @@ -94,17 +94,16 @@ const trtllmAdapter: ServerMetricsAdapter = { : 'unknown'; const endpointUrl = series.endpoint_url ?? null; const workerId = labels['worker_id'] ?? null; - const dpRank = labels['dp_rank'] ?? null; - const engine = labels['engine'] ?? labels['engine_idx'] ?? null; return { - id: stableId('trtllm', [role, endpointUrl, workerId, dpRank, engine]), + // Native TRT metrics and Dynamo's rank-labelled gauges share one endpoint. + id: stableId('trtllm', [role, endpointUrl ?? workerId]), adapter: 'trtllm', role, endpointUrl, nativeRole, workerId, - dpRank, - engine, + dpRank: null, + engine: null, }; }, }; From 0896009ec057b8b3dbf718e17467f1cceb14a4dd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 09:48:04 -0500 Subject: [PATCH 10/10] Add run-scoped server metrics recomputation --- .../workflows/recompute-agentic-metrics.yml | 48 +++++++++++++++++++ .github/workflows/stage-results.yml | 2 +- docs/data-pipeline.md | 5 ++ packages/db/src/backfill-aggregate-stats.ts | 28 +++++++++-- packages/db/src/backfill-chart-series.ts | 8 +--- packages/db/src/lib/backfill-runner.test.ts | 22 +++++++++ packages/db/src/lib/backfill-runner.ts | 12 +++++ 7 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/recompute-agentic-metrics.yml diff --git a/.github/workflows/recompute-agentic-metrics.yml b/.github/workflows/recompute-agentic-metrics.yml new file mode 100644 index 000000000..6550e7192 --- /dev/null +++ b/.github/workflows/recompute-agentic-metrics.yml @@ -0,0 +1,48 @@ +name: Recompute Agentic Metrics +run-name: Recompute production metrics for run ${{ inputs.run-id }} + +on: + workflow_dispatch: + inputs: + run-id: + description: Already-ingested GitHub benchmark run to recompute in production + required: true + type: string + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ inputs.run-id }} + cancel-in-progress: false + +jobs: + recompute: + name: Recompute stored server metrics + if: github.ref == 'refs/heads/master' + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 90 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + - name: Recompute chart series and aggregate statistics + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} + RUN_ID: ${{ inputs.run-id }} + run: | + bun run --cwd packages/db db:backfill-chart-series --run-id "$RUN_ID" --force --yes + bun run --cwd packages/db db:backfill-aggregate-stats --run-id "$RUN_ID" --force --yes + - name: Invalidate production cache + env: + INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} + run: bun run admin:cache:invalidate https://inferencex.semianalysis.com diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index b38576f53..7079abe4d 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -254,7 +254,7 @@ jobs: needs: [validate, prepare-staging-database] permissions: contents: read - uses: ./.github/workflows/ingest-agentic-results.yml + uses: $/.github/workflows/ingest-agentic-results.yml with: run-id: ${{ needs.validate.outputs.run-id }} run-attempt: ${{ needs.validate.outputs.run-attempt }} diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index c7087e869..8b1daa2bd 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -177,6 +177,11 @@ retains `dp_rank`, over the rankless native `trtllm_kv_cache_utilization` gauge. Ranks remain inside each worker's source; native-only endpoints retain their fallback. The all-endpoints view summarizes workers, while selecting a worker exposes its ranks. +For runs ingested before a server-metric adapter was available, dispatch +`Recompute Agentic Metrics` on `master` with the benchmark run ID. It force-rebuilds +chart series and aggregate statistics from stored blobs for that run in production, +then invalidates the app cache. It does not rerun benchmarks or replace raw artifacts. + ### Logical Engines vs Raw Series A raw series in the blob is one `(scrape endpoint × phase block × label set)` tuple, which is **not** the same as one engine. The KV-cache chart needs one entry per _logical engine_ — one KV pool — so `compute-chart-series.ts` groups series by their Prometheus label set (`seriesIdentityKey`) rather than emitting one entry per raw series. Three kinds of duplication collapse there: diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index c1ce8f2f0..f71d7839a 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -18,6 +18,7 @@ * [--limit N] only process the first N candidate rows (useful for * smoke-tests on a fresh deploy) * [--force] recompute every row, even if version already matches + * [--run-id N] only process trace rows linked to one GitHub workflow run * [--yes] skip the confirmation prompt */ @@ -33,11 +34,13 @@ import { createAdminSql } from './etl/db-utils.js'; import { jsonbParam, parseLimitForceFlags, + parseRunIdFlag, runBackfillMain, runCandidateIdBackfill, } from './lib/backfill-runner.js'; const flags = parseLimitForceFlags(); +const githubRunId = parseRunIdFlag(); // Neon's HTTP response and JS drivers should not receive a 100+ MB bytea as // one value. Slice oversized TOAST values into independent bounded reads and @@ -69,9 +72,21 @@ async function main(): Promise { console.log(` STATS_VERSION = ${STATS_VERSION}`); console.log(` force = ${flags.force}`); console.log(` limit = ${flags.limit ?? 'none'}`); + console.log(` run_id = ${githubRunId ?? 'all'}`); await runCandidateIdBackfill( async () => { + const runFilter = githubRunId + ? sql` + and exists ( + select 1 + from benchmark_results br + join latest_workflow_runs wr on wr.id = br.workflow_run_id + where br.trace_replay_id = agentic_trace_replay.id + and wr.github_run_id = ${githubRunId} + ) + ` + : sql``; // Find candidates: rows missing stats, or whose stored version is stale. // Using >>'version'::int comparison would error on null; coalesce to -1 so // null-stats rows always count as stale. @@ -79,14 +94,16 @@ async function main(): Promise { ? await sql<{ id: number }[]>` select id from agentic_trace_replay + where true ${runFilter} order by id ${flags.limit ? sql`limit ${flags.limit}` : sql``} ` : await sql<{ id: number }[]>` select id from agentic_trace_replay - where aggregate_stats is null - or coalesce((aggregate_stats->>'version')::int, -1) <> ${STATS_VERSION} + where (aggregate_stats is null + or coalesce((aggregate_stats->>'version')::int, -1) <> ${STATS_VERSION}) + ${runFilter} order by id ${flags.limit ? sql`limit ${flags.limit}` : sql``} `; @@ -132,7 +149,12 @@ async function main(): Promise { // fields haven't changed since v3), so skip re-reading the huge server // blob and carry its KV/prefix distributions forward. const storedVersion = row.aggregate_stats?.version; - if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { + if ( + !flags.force && + storedVersion !== undefined && + storedVersion >= 3 && + storedVersion < STATS_VERSION + ) { stats = mergeProfileStatsUpgrade(row.aggregate_stats!, profileStats); } else { const [serverRow] = await sql<{ server_metrics_json_gz: Buffer | null }[]>` diff --git a/packages/db/src/backfill-chart-series.ts b/packages/db/src/backfill-chart-series.ts index 10f58503f..12a04ae4d 100644 --- a/packages/db/src/backfill-chart-series.ts +++ b/packages/db/src/backfill-chart-series.ts @@ -31,17 +31,13 @@ import { createAdminSql } from './etl/db-utils.js'; import { jsonbParam, parseLimitForceFlags, + parseRunIdFlag, runBackfillMain, runCandidateIdBackfill, } from './lib/backfill-runner.js'; const flags = parseLimitForceFlags(); -const runIdIndex = process.argv.indexOf('--run-id'); -const runIdRaw = runIdIndex === -1 ? undefined : process.argv[runIdIndex + 1]; -if (runIdIndex !== -1 && (!runIdRaw || !/^\d+$/u.test(runIdRaw))) { - throw new Error('--run-id must be followed by a numeric GitHub workflow run ID'); -} -const githubRunId = runIdRaw ? Number(runIdRaw) : undefined; +const githubRunId = parseRunIdFlag(); const sql = createAdminSql({ noSsl: hasNoSslFlag(), diff --git a/packages/db/src/lib/backfill-runner.test.ts b/packages/db/src/lib/backfill-runner.test.ts index cf6b40f9c..24cbe5fe4 100644 --- a/packages/db/src/lib/backfill-runner.test.ts +++ b/packages/db/src/lib/backfill-runner.test.ts @@ -2,10 +2,32 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { parseLimitForceFlags, + parseRunIdFlag, runCandidateIdBackfill, runPerIdBackfill, } from './backfill-runner.js'; +describe('parseRunIdFlag', () => { + it('leaves unscoped backfills unchanged', () => { + expect(parseRunIdFlag(['bun', 'backfill.ts', '--force'])).toBeUndefined(); + }); + + it('preserves large GitHub run IDs', () => { + expect(parseRunIdFlag(['bun', 'backfill.ts', '--run-id', '33418433573', '--yes'])).toBe( + 33418433573, + ); + }); + + it.each([undefined, '', '--yes', 'all', '0', '-1', '1.5', '1e3', '9007199254740992'])( + 'rejects an invalid run selector: %s', + (value) => { + const argv = ['bun', 'backfill.ts', '--run-id']; + if (value !== undefined) argv.push(value); + expect(() => parseRunIdFlag(argv)).toThrow('--run-id requires a positive integer'); + }, + ); +}); + describe('parseLimitForceFlags', () => { const originalArgv = process.argv; afterEach(() => { diff --git a/packages/db/src/lib/backfill-runner.ts b/packages/db/src/lib/backfill-runner.ts index 5fde13ede..fd92b8a1d 100644 --- a/packages/db/src/lib/backfill-runner.ts +++ b/packages/db/src/lib/backfill-runner.ts @@ -16,6 +16,18 @@ export interface LimitForceFlags { shardIndex: number; } +/** Restrict a backfill to one GitHub run; malformed selectors must never scan all rows. */ +export function parseRunIdFlag(argv: readonly string[] = process.argv): number | undefined { + const index = argv.indexOf('--run-id'); + if (index === -1) return undefined; + const raw = argv[index + 1]; + const runId = Number(raw); + if (!raw || !/^\d+$/u.test(raw) || !Number.isSafeInteger(runId) || runId < 1) { + throw new Error('--run-id requires a positive integer GitHub workflow run ID'); + } + return runId; +} + /** Parse the standard `--limit N` / `--force` backfill flags from argv. */ export function parseLimitForceFlags(): LimitForceFlags { let limit: number | null = null;