diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index e9e273777..6e23cd53b 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -41,6 +41,16 @@ on: required: false default: production type: string + neon-branch: + description: Named Neon branch when database-target is neon-branch + required: false + default: '' + type: string + preview-url: + description: Preview deployment URL to invalidate for a named Neon branch + required: false + default: '' + type: string secrets: DATABASE_WRITE_URL: description: Production database write connection for direct ingests @@ -60,6 +70,12 @@ on: INFX_MAIN_PAT: description: Token used to download InferenceX artifacts required: true + NEON_API_KEY: + description: Neon API key used to resolve named child branches + required: false + NEON_PROJECT_ID: + description: Neon project containing named child branches + required: false SLACK_WEBHOOK_URL: description: Optional failure and unmapped-entity notifications required: false @@ -83,6 +99,17 @@ on: - production - dev - staging + - neon-branch + neon-branch: + description: Named Neon branch when database-target is neon-branch + required: false + default: '' + type: string + preview-url: + description: Preview deployment URL to invalidate for a named Neon branch + required: false + default: '' + type: string permissions: {} @@ -148,6 +175,10 @@ jobs: PROTECTION_BYPASS_SECRET_STAGING: ${{ secrets.VERCEL_STAGING_BYPASS_SECRET }} STAGING_SITE_URL: ${{ vars.STAGING_SITE_URL || 'https://inferencemax-app-git-staging-semianalysisai.vercel.app' }} + NEON_API_KEY: ${{ secrets.NEON_API_KEY }} + NEON_PROJECT_ID: ${{ secrets.NEON_PROJECT_ID }} + NEON_BRANCH_NAME: ${{ inputs.neon-branch }} + PREVIEW_SITE_URL: ${{ inputs.preview-url }} run: | case "$REQUESTED_DATABASE_TARGET" in production) @@ -165,6 +196,40 @@ jobs: cache_invalidate_url="${STAGING_SITE_URL%/}/api/v1/invalidate" protection_bypass_secret="$PROTECTION_BYPASS_SECRET_STAGING" ;; + neon-branch) + if [ -z "$NEON_API_KEY" ] || [ -z "$NEON_PROJECT_ID" ] || [ -z "$NEON_BRANCH_NAME" ] || [ -z "$PREVIEW_SITE_URL" ]; then + echo "::error::NEON_API_KEY, NEON_PROJECT_ID, neon-branch, and preview-url are required" + exit 1 + fi + branches=$(curl --retry 3 --retry-all-errors -sSf \ + -H "Authorization: Bearer $NEON_API_KEY" \ + "https://console.neon.tech/api/v2/projects/$NEON_PROJECT_ID/branches?limit=100") + branch_id=$(jq -r --arg name "$NEON_BRANCH_NAME" \ + '.branches[] | select(.name == $name and .parent_id != null and .default != true) | .id' <<<"$branches" | head -n 1) + if [ -z "$branch_id" ] || [ "$branch_id" = "null" ]; then + echo "::error::Neon branch not found: $NEON_BRANCH_NAME" + exit 1 + fi + database_write_url=$(npx --yes neonctl connection-string "$branch_id" \ + --api-key "$NEON_API_KEY" \ + --project-id "$NEON_PROJECT_ID" \ + --database-name neondb \ + --role-name neondb_owner) + endpoints=$(curl --retry 3 --retry-all-errors -sSf \ + -H "Authorization: Bearer $NEON_API_KEY" \ + "https://console.neon.tech/api/v2/projects/$NEON_PROJECT_ID/endpoints") + database_host=$(DATABASE_URL_TO_CHECK="$database_write_url" node -e \ + 'console.log(new URL(process.env.DATABASE_URL_TO_CHECK).hostname)') + if ! jq -e --arg branch "$branch_id" --arg host "$database_host" \ + '.endpoints[] | select(.branch_id == $branch and .host == $host)' <<<"$endpoints" >/dev/null; then + echo "::error::Resolved database host does not belong to the requested child branch" + exit 1 + fi + echo "Verified child database: $NEON_BRANCH_NAME ($branch_id), host $database_host" + cache_invalidate_url="${PREVIEW_SITE_URL%/}/api/v1/invalidate" + cache_invalidate_secret="$INVALIDATE_SECRET_DEFAULT" + protection_bypass_secret="$PROTECTION_BYPASS_SECRET_STAGING" + ;; *) echo "::error::Unsupported database-target: $REQUESTED_DATABASE_TARGET" exit 1 @@ -175,12 +240,12 @@ jobs: echo "::error::Database secret is empty for target: $REQUESTED_DATABASE_TARGET" exit 1 fi - if [ "$REQUESTED_DATABASE_TARGET" != "staging" ] && [ -z "$cache_invalidate_secret" ]; then + if [ "$REQUESTED_DATABASE_TARGET" != "staging" ] && [ "$REQUESTED_DATABASE_TARGET" != "neon-branch" ] && [ -z "$cache_invalidate_secret" ]; then echo "::error::Cache invalidation secret is empty for target: $REQUESTED_DATABASE_TARGET" exit 1 fi - if [ "$REQUESTED_DATABASE_TARGET" = "staging" ] && [ -z "$protection_bypass_secret" ]; then - echo "::error::Vercel protection bypass secret is empty for staging" + if { [ "$REQUESTED_DATABASE_TARGET" = "staging" ] || [ "$REQUESTED_DATABASE_TARGET" = "neon-branch" ]; } && [ -z "$protection_bypass_secret" ]; then + echo "::error::Vercel protection bypass secret is empty for preview target" exit 1 fi @@ -237,6 +302,11 @@ jobs: if [ "$INGEST_DATABASE_TARGET" = "staging" ]; then curl --retry 3 --retry-delay 2 --retry-connrefused -sSf -X POST "$CACHE_INVALIDATE_URL" \ -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" + elif [ "$INGEST_DATABASE_TARGET" = "neon-branch" ]; then + curl --retry 3 --retry-delay 2 --retry-connrefused -sSf -X POST "$CACHE_INVALIDATE_URL" \ + -H "x-vercel-protection-bypass: $CACHE_PROTECTION_BYPASS_SECRET" \ + -H "Authorization: Bearer $CACHE_INVALIDATE_SECRET" || \ + echo "::warning::Preview cache invalidation failed; the database ingest is complete" else curl -sSf -X POST "$CACHE_INVALIDATE_URL" \ -H "Authorization: Bearer $CACHE_INVALIDATE_SECRET" || true diff --git a/.github/workflows/recompute-agentic-metrics.yml b/.github/workflows/recompute-agentic-metrics.yml new file mode 100644 index 000000000..6550e7192 --- /dev/null +++ b/.github/workflows/recompute-agentic-metrics.yml @@ -0,0 +1,48 @@ +name: Recompute Agentic Metrics +run-name: Recompute production metrics for run ${{ inputs.run-id }} + +on: + workflow_dispatch: + inputs: + run-id: + description: Already-ingested GitHub benchmark run to recompute in production + required: true + type: string + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ inputs.run-id }} + cancel-in-progress: false + +jobs: + recompute: + name: Recompute stored server metrics + if: github.ref == 'refs/heads/master' + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 90 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + - name: Recompute chart series and aggregate statistics + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} + RUN_ID: ${{ inputs.run-id }} + run: | + bun run --cwd packages/db db:backfill-chart-series --run-id "$RUN_ID" --force --yes + bun run --cwd packages/db db:backfill-aggregate-stats --run-id "$RUN_ID" --force --yes + - name: Invalidate production cache + env: + INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} + run: bun run admin:cache:invalidate https://inferencex.semianalysis.com diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index 9f91a765c..7079abe4d 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -254,7 +254,7 @@ jobs: needs: [validate, prepare-staging-database] permissions: contents: read - uses: ./.github/workflows/ingest-agentic-results.yml + uses: $/.github/workflows/ingest-agentic-results.yml with: run-id: ${{ needs.validate.outputs.run-id }} run-attempt: ${{ needs.validate.outputs.run-attempt }} @@ -265,10 +265,34 @@ jobs: INFX_MAIN_PAT: ${{ secrets.INFX_MAIN_PAT }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + refresh-chart-series: + name: Refresh staged chart series + needs: [validate, ingest] + runs-on: blacksmith-16vcpu-ubuntu-2404 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + - name: Recompute stale chart series for staged run + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_STAGING_WRITE_URL }} + RUN_ID: ${{ needs.validate.outputs.run-id }} + run: bun run --cwd packages/db db:backfill-chart-series --run-id "$RUN_ID" --yes + report: name: Report staging result if: ${{ always() && needs.validate.result == 'success' }} - needs: [validate, sync-staging-branch, prepare-staging-database, ingest] + needs: [validate, sync-staging-branch, prepare-staging-database, ingest, refresh-chart-series] runs-on: ubuntu-latest permissions: {} steps: @@ -280,7 +304,7 @@ jobs: PR_NUMBER: ${{ needs.validate.outputs.pr-number }} REQUESTED_BY: ${{ needs.validate.outputs.requested-by }} COMMENT_ID: ${{ needs.validate.outputs.comment-id }} - STAGE_SUCCEEDED: ${{ needs['sync-staging-branch'].result == 'success' && needs['prepare-staging-database'].result == 'success' && needs.ingest.result == 'success' }} + STAGE_SUCCEEDED: ${{ needs['sync-staging-branch'].result == 'success' && needs['prepare-staging-database'].result == 'success' && needs.ingest.result == 'success' && needs['refresh-chart-series'].result == 'success' }} with: github-token: ${{ secrets.PAT }} script: | diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 176acc32c..8b1daa2bd 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -172,6 +172,16 @@ AIPerf defines the `server_metrics_export.json` envelope, but labels such as wor Adapters are selected from the benchmark's canonical framework, and per-worker series are only emitted for disaggregated configs with a recognized adapter. Unknown orchestrators and non-disaggregated configs retain their aggregate-only series; roles are never guessed from ports or metric names. The frontend only consumes the canonical source identity and never interprets orchestrator-native labels. +Dynamo/TRT-LLM KV charts prefer `dynamo_component_gpu_cache_usage_percent`, which +retains `dp_rank`, over the rankless native `trtllm_kv_cache_utilization` gauge. +Ranks remain inside each worker's source; native-only endpoints retain their fallback. +The all-endpoints view summarizes workers, while selecting a worker exposes its ranks. + +For runs ingested before a server-metric adapter was available, dispatch +`Recompute Agentic Metrics` on `master` with the benchmark run ID. It force-rebuilds +chart series and aggregate statistics from stored blobs for that run in production, +then invalidates the app cache. It does not rerun benchmarks or replace raw artifacts. + ### Logical Engines vs Raw Series A raw series in the blob is one `(scrape endpoint × phase block × label set)` tuple, which is **not** the same as one engine. The KV-cache chart needs one entry per _logical engine_ — one KV pool — so `compute-chart-series.ts` groups series by their Prometheus label set (`seriesIdentityKey`) rather than emitting one entry per raw series. Three kinds of duplication collapse there: diff --git a/packages/app/src/components/inference/agentic-point/chart-legend.test.ts b/packages/app/src/components/inference/agentic-point/chart-legend.test.ts new file mode 100644 index 000000000..be475d90a --- /dev/null +++ b/packages/app/src/components/inference/agentic-point/chart-legend.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; + +import { + LEGEND_ITEM_GAP, + LEGEND_ROW_HEIGHT, + LEGEND_TEXT_OFFSET, + estimateTextWidth, + layoutChartLegend, +} from './chart-legend'; + +// Inline render of the point-detail charts: 720 viewBox units wide with the +// shared 60/16 left/right axis padding. +const INLINE_INNER_WIDTH = 720 - 60 - 16; + +/** Right edge of the item at `index`, i.e. where its text stops. */ +const itemRight = (labels: readonly string[], index: number, width: number): number => { + const { items } = layoutChartLegend(labels, width); + return items[index]!.x + LEGEND_TEXT_OFFSET + estimateTextWidth(labels[index]!); +}; + +describe('estimateTextWidth', () => { + it('grows with the number of characters', () => { + expect(estimateTextWidth('decode')).toBeGreaterThan(estimateTextWidth('dec')); + }); + + it('counts CJK characters as roughly one em each', () => { + // Four fullwidth characters at font size 11. + expect(estimateTextWidth('芯片缓存')).toBeCloseTo(44, 5); + }); + + it('charges more for wide characters than narrow ones', () => { + expect(estimateTextWidth('mmmm')).toBeGreaterThan(estimateTextWidth('llll')); + }); + + it('returns zero for an empty label', () => { + expect(estimateTextWidth('')).toBe(0); + }); + + it('scales linearly with font size', () => { + expect(estimateTextWidth('decode', 22)).toBeCloseTo(estimateTextWidth('decode', 11) * 2, 5); + }); +}); + +describe('layoutChartLegend', () => { + it('keeps a short legend on one row and reserves no extra height', () => { + const layout = layoutChartLegend(['Input', 'Decode'], INLINE_INNER_WIDTH); + expect(layout.rows).toBe(1); + expect(layout.extraHeight).toBe(0); + expect(layout.items.map((i) => i.row)).toEqual([0, 0]); + }); + + it('lays same-row items out left to right without overlapping', () => { + const labels = ['Input', 'Decode']; + const { items } = layoutChartLegend(labels, INLINE_INNER_WIDTH); + expect(items[0]!.x).toBe(0); + expect(items[1]!.x).toBe(LEGEND_TEXT_OFFSET + estimateTextWidth(labels[0]!) + LEGEND_ITEM_GAP); + expect(items[1]!.x).toBeGreaterThanOrEqual(itemRight(labels, 0, INLINE_INNER_WIDTH)); + }); + + it('wraps the eleven-series KV-cache legend instead of overprinting it', () => { + // The regression from the inline KV-cache chart: eleven per-engine labels + // in ~644 units used to get 58 units each and collide. + const labels = [ + 'prefill (500e)', + 'prefill (501a)', + 'prefill (501e)', + 'decode (5021)', + 'decode (5023)', + 'decode (5025)', + 'decode (5027)', + 'Chip HBM (avg n=50)', + 'DRAM', + 'CPU offload pool (avg n=50)', + 'Avg', + ]; + const layout = layoutChartLegend(labels, INLINE_INNER_WIDTH); + + expect(layout.rows).toBeGreaterThan(1); + expect(layout.extraHeight).toBe((layout.rows - 1) * LEGEND_ROW_HEIGHT); + + for (const [i, label] of labels.entries()) { + const item = layout.items[i]!; + const right = item.x + LEGEND_TEXT_OFFSET + estimateTextWidth(label); + // Every item fits inside the plot width... + expect(right).toBeLessThanOrEqual(INLINE_INNER_WIDTH); + // ...and starts clear of its predecessor on the same row. + const prev = layout.items[i - 1]; + if (prev && prev.row === item.row) { + expect(item.x).toBeGreaterThanOrEqual( + prev.x + LEGEND_TEXT_OFFSET + estimateTextWidth(labels[i - 1]!), + ); + } + } + }); + + it('starts each wrapped row back at the left edge', () => { + const labels = Array.from({ length: 12 }, (_, i) => `series number ${i}`); + const layout = layoutChartLegend(labels, INLINE_INNER_WIDTH); + const firstOfRow = new Map(); + for (const item of layout.items) { + if (!firstOfRow.has(item.row)) firstOfRow.set(item.row, item.x); + } + for (const x of firstOfRow.values()) expect(x).toBe(0); + }); + + it('wraps sooner for Chinese labels, which are wider per character', () => { + const en = Array.from({ length: 6 }, () => 'Chip HBM pool'); + const zh = Array.from({ length: 6 }, () => '芯片 HBM 显存池均值'); + expect(layoutChartLegend(zh, INLINE_INNER_WIDTH).rows).toBeGreaterThanOrEqual( + layoutChartLegend(en, INLINE_INNER_WIDTH).rows, + ); + }); + + it('needs fewer rows at the expanded width than inline', () => { + const labels = Array.from({ length: 11 }, (_, i) => `decode engine ${i}`); + const inline = layoutChartLegend(labels, INLINE_INNER_WIDTH); + const expanded = layoutChartLegend(labels, 1300 - 60 - 16); + expect(expanded.rows).toBeLessThan(inline.rows); + }); + + it('gives a label wider than the row its own row rather than dropping it', () => { + const layout = layoutChartLegend(['a', 'w'.repeat(300), 'b'], 100); + expect(layout.items).toHaveLength(3); + expect(layout.items[1]!.row).toBe(1); + expect(layout.items[1]!.x).toBe(0); + expect(layout.items[2]!.row).toBe(2); + }); + + it('reports a single row for an empty legend', () => { + const layout = layoutChartLegend([], INLINE_INNER_WIDTH); + expect(layout.items).toEqual([]); + expect(layout.rows).toBe(1); + expect(layout.extraHeight).toBe(0); + }); +}); diff --git a/packages/app/src/components/inference/agentic-point/chart-legend.ts b/packages/app/src/components/inference/agentic-point/chart-legend.ts new file mode 100644 index 000000000..5dfc72dfd --- /dev/null +++ b/packages/app/src/components/inference/agentic-point/chart-legend.ts @@ -0,0 +1,122 @@ +/** + * Legend layout for the hand-rolled agentic point-detail charts. + * + * Those charts are plain SVG inside a fixed viewBox that scales to the card + * width, so there is no DOM to measure text against at layout time. The legend + * used to sit on an equal-width grid (`innerW / itemCount`), which collides as + * soon as the labels are longer than their slot — the inline 720-unit render of + * the KV-cache chart packs eleven per-engine series into ~58 units each and the + * labels overprint one another. + * + * Instead we estimate each label's advance width from its characters and pack + * items greedily into as many rows as they need. Estimates are deliberately a + * little generous: over-estimating wraps one item early, under-estimating + * reintroduces the overlap this module exists to prevent. + */ + +/** Font size the chart legends render at. */ +export const LEGEND_FONT_SIZE = 11; + +/** Baseline-to-baseline distance between wrapped legend rows. */ +export const LEGEND_ROW_HEIGHT = 14; + +/** x offset from an item's origin to the start of its color swatch. */ +export const LEGEND_SWATCH_INSET = 2; + +/** Width of the color swatch (line segment or filled rect). */ +export const LEGEND_SWATCH_WIDTH = 12; + +/** x offset from an item's origin to the start of its text. */ +export const LEGEND_TEXT_OFFSET = 18; + +/** Horizontal gap between two legend items on the same row. */ +export const LEGEND_ITEM_GAP = 12; + +/** Distance from the bottom of the viewBox to the last legend row's baseline. */ +export const LEGEND_BASELINE_OFFSET = 8; + +// Fullwidth scripts (CJK ideographs, kana, Hangul, fullwidth forms) advance +// roughly one em per character rather than the ~0.5em of Latin lowercase. +const FULLWIDTH = + /[\u1100-\u115F\u2E80-\u303E\u3041-\u33FF\u3400-\u4DBF\u4E00-\u9FFF\uA000-\uA4CF\uAC00-\uD7A3\uF900-\uFAFF\uFE30-\uFE4F\uFF00-\uFF60\uFFE0-\uFFE6]/u; + +// Character classes, in em units at the legend font size. Sampled against the +// app's sans stack and rounded up. +const NARROW_CHARS = new Set(" .,:;!|'`ijltfr()[]{}/\\-"); +const WIDE_CHARS = new Set('mwMW@%'); +const UPPER_OR_DIGIT = /[A-Z0-9$#]/u; + +const EM_FULLWIDTH = 1; +const EM_NARROW = 0.34; +const EM_WIDE = 0.88; +const EM_UPPER_OR_DIGIT = 0.62; +const EM_DEFAULT = 0.53; + +/** + * Approximate rendered width of `text` in viewBox units. + * + * Not exact — SVG text has no measurable width until it is in the document — + * but consistently at or slightly above the real advance width, which is the + * side to err on for collision avoidance. + */ +export function estimateTextWidth(text: string, fontSize: number = LEGEND_FONT_SIZE): number { + let em = 0; + for (const ch of text) { + if (FULLWIDTH.test(ch)) em += EM_FULLWIDTH; + else if (NARROW_CHARS.has(ch)) em += EM_NARROW; + else if (WIDE_CHARS.has(ch)) em += EM_WIDE; + else if (UPPER_OR_DIGIT.test(ch)) em += EM_UPPER_OR_DIGIT; + else em += EM_DEFAULT; + } + return em * fontSize; +} + +/** Placement of one legend item within the wrapped legend block. */ +export interface LegendItemLayout { + /** x of the item's origin, relative to the left edge of the plot area. */ + x: number; + /** 0-based row the item was packed into; row 0 is the topmost. */ + row: number; +} + +export interface LegendLayout { + items: LegendItemLayout[]; + /** Number of rows the legend occupies. At least 1, even when empty. */ + rows: number; + /** + * Vertical space, in viewBox units, the legend needs beyond the single row + * the chart already reserves in its bottom padding. Charts add this to both + * their height and their bottom padding so the plot area is unchanged and + * the extra rows extend the SVG downward. + */ + extraHeight: number; +} + +/** + * Pack legend labels into rows no wider than `availableWidth`. + * + * A label wider than the whole row still gets its own row rather than being + * dropped — a clipped label is more useful than a missing one. + */ +export function layoutChartLegend( + labels: readonly string[], + availableWidth: number, + fontSize: number = LEGEND_FONT_SIZE, +): LegendLayout { + const items: LegendItemLayout[] = []; + let row = 0; + let cursor = 0; + + for (const label of labels) { + const itemWidth = LEGEND_TEXT_OFFSET + estimateTextWidth(label, fontSize); + if (cursor > 0 && cursor + itemWidth > availableWidth) { + row += 1; + cursor = 0; + } + items.push({ x: cursor, row }); + cursor += itemWidth + LEGEND_ITEM_GAP; + } + + const rows = items.length === 0 ? 1 : row + 1; + return { items, rows, extraHeight: (rows - 1) * LEGEND_ROW_HEIGHT }; +} diff --git a/packages/app/src/components/inference/agentic-point/chart-shared.tsx b/packages/app/src/components/inference/agentic-point/chart-shared.tsx index b8333b4e9..8379327d2 100644 --- a/packages/app/src/components/inference/agentic-point/chart-shared.tsx +++ b/packages/app/src/components/inference/agentic-point/chart-shared.tsx @@ -2,6 +2,16 @@ import { useLocale } from '@/lib/use-locale'; +import { + LEGEND_BASELINE_OFFSET, + LEGEND_FONT_SIZE, + LEGEND_ROW_HEIGHT, + LEGEND_SWATCH_INSET, + LEGEND_SWATCH_WIDTH, + LEGEND_TEXT_OFFSET, + type LegendLayout, +} from './chart-legend'; + /** * Shared presentational constants and helpers for the agentic point-detail * charts (time-series, stacked-area, distribution, aggregate). These charts @@ -58,3 +68,88 @@ export function ChartEmpty({ height = 260, message }: { height?: number; message export function ChartSkeleton() { return
; } + +/** How a legend entry's color is shown: a stroked line or a filled block. */ +export type LegendSwatch = 'line' | 'dashed-line' | 'area'; + +export interface ChartLegendEntry { + label: string; + color: string; + /** Defaults to `'line'`. */ + swatch?: LegendSwatch; + /** Stroke width for the line swatches. Defaults to 2. */ + strokeWidth?: number; + /** Swatch opacity, to match a translucent area fill. Defaults to 1. */ + opacity?: number; +} + +/** + * Wrapped legend for the hand-rolled point-detail charts. + * + * `layout` comes from `layoutChartLegend()` and must have been computed from + * the same entry order. `baselineY` is the baseline of the *last* row (charts + * pass `height - LEGEND_BASELINE_OFFSET`), so a single-row legend lands exactly + * where it did before wrapping existed and extra rows grow downward into the + * space the chart added to its bottom padding. + */ +export function ChartLegend({ + entries, + layout, + left, + baselineY, +}: { + entries: readonly ChartLegendEntry[]; + layout: LegendLayout; + /** x of the plot area's left edge; item x offsets are relative to it. */ + left: number; + baselineY: number; +}) { + const topRowBaseline = baselineY - (layout.rows - 1) * LEGEND_ROW_HEIGHT; + return ( + <> + {entries.map((entry, i) => { + const placed = layout.items[i]; + if (!placed) return null; + const x = left + placed.x; + const y = topRowBaseline + placed.row * LEGEND_ROW_HEIGHT; + const swatch = entry.swatch ?? 'line'; + return ( + + {swatch === 'area' ? ( + + ) : ( + + )} + + {entry.label} + + + ); + })} + + ); +} + +export { LEGEND_BASELINE_OFFSET }; diff --git a/packages/app/src/components/inference/agentic-point/distribution.tsx b/packages/app/src/components/inference/agentic-point/distribution.tsx index f11d9f20b..f62a61e5e 100644 --- a/packages/app/src/components/inference/agentic-point/distribution.tsx +++ b/packages/app/src/components/inference/agentic-point/distribution.tsx @@ -5,7 +5,16 @@ import { useMemo } from 'react'; import { useLocale } from '@/lib/use-locale'; import { ChartHover, type HoverItem } from './chart-hover'; -import { CHART_PAD, ChartEmpty, PERCENTILE_COLORS, fmtCount } from './chart-shared'; +import { + CHART_PAD, + ChartEmpty, + ChartLegend, + LEGEND_BASELINE_OFFSET, + PERCENTILE_COLORS, + fmtCount, + type ChartLegendEntry, +} from './chart-shared'; +import { layoutChartLegend } from './chart-legend'; import { logHistogram, logTicks, positiveValues } from './lognormal'; import { quantile } from './time-series-math'; @@ -69,7 +78,6 @@ export function Distribution({ }) { const t = STRINGS[useLocale()]; const W = width; - const H = height; const computed = useMemo(() => { const positive = positiveValues(values); @@ -83,16 +91,31 @@ export function Distribution({ // Zero-token requests are real but unplottable on a log axis; they are // reported under the chart rather than silently folded into bin one. excluded: values.length - positive.length, - innerW: W - PAD.left - PAD.right, - innerH: H - PAD.top - PAD.bottom, }; - }, [values, W, H]); + }, [values]); if (!computed) { - return ; + return ; } - const { sorted, histogram, excluded, innerW, innerH } = computed; + const { sorted, histogram, excluded } = computed; const { counts, edges, lnMin, lnMax } = histogram; + + // Wrap the legend, then grow the viewBox and bottom padding by the same + // amount so the plot area is untouched (see TimeSeriesChart). + const legendEntries: ChartLegendEntry[] = GUIDES.map(({ label, q, color }) => ({ + label: `${label} ${fmtCount(quantile(sorted, q))}`, + color, + swatch: 'dashed-line', + })); + const innerW = W - PAD.left - PAD.right; + const legend = layoutChartLegend( + legendEntries.map((e) => e.label), + innerW, + ); + const H = height + legend.extraHeight; + const padBottom = PAD.bottom + legend.extraHeight; + const pad = { ...PAD, bottom: padBottom }; + const innerH = H - PAD.top - padBottom; const min = edges[0]!; const max = edges.at(-1)!; const nBins = counts.length; @@ -135,7 +158,7 @@ export function Distribution({ {sorted.length.toLocaleString()} {t.requests} · {t.range} {fmt(min)}–{fmt(max)} {unit} ·{' '} {t.logScale}
- + {/* y-axis gridlines + labels */} {yTickVals.map((v, i) => { const y = yScale(v); @@ -227,7 +250,7 @@ export function Distribution({ })} {t.countAxis} {/* Percentile legend chips */} - {(() => { - const chipY = H - 8; - const chipW = innerW / GUIDES.length; - return GUIDES.map(({ label: ql, q, color }, i) => { - const x = PAD.left + i * chipW; - return ( - - - - {ql} {fmt(quantile(sorted, q))} - - - ); - }); - })()} + {excluded > 0 && (
diff --git a/packages/app/src/components/inference/agentic-point/point-summary.test.ts b/packages/app/src/components/inference/agentic-point/point-summary.test.ts index 7fef6f8d0..4dbfd8288 100644 --- a/packages/app/src/components/inference/agentic-point/point-summary.test.ts +++ b/packages/app/src/components/inference/agentic-point/point-summary.test.ts @@ -68,6 +68,23 @@ describe('PointSummary', () => { expect(html).toContain('42.00%'); }); + it('labels TRTLLM offload cache hits as combined instead of showing an empty CPU rate', () => { + const html = renderToStaticMarkup( + createElement(PointSummary, { + meta: meta({ + framework: 'dynamo-trt', + kv_offloading: 'dram', + server_gpu_cache_hit_rate: 0.978, + server_cpu_cache_hit_rate: null, + }), + }), + ); + + expect(html).toContain('Combined chip + CPU cache hit'); + expect(html).toContain('97.80%'); + expect(html).not.toContain('>CPU cache hit<'); + }); + it('shows runtime component names and independently reported versions', () => { const html = renderToStaticMarkup( createElement(PointSummary, { diff --git a/packages/app/src/components/inference/agentic-point/point-summary.tsx b/packages/app/src/components/inference/agentic-point/point-summary.tsx index 1bd6b3a0d..99c10cf2b 100644 --- a/packages/app/src/components/inference/agentic-point/point-summary.tsx +++ b/packages/app/src/components/inference/agentic-point/point-summary.tsx @@ -3,6 +3,7 @@ import type { ReactNode } from 'react'; import type { PointMeta } from '@/hooks/api/use-trace-server-metrics'; +import { frameworkFamily } from '@/lib/framework-family'; import type { Locale } from '@/lib/i18n'; import { isKvOffloadEnabled } from '@/lib/kv-offload'; import { useLocale } from '@/lib/use-locale'; @@ -24,6 +25,7 @@ export const POINT_SUMMARY_STRINGS = { concurrency: 'Concurrency', gpuCacheHit: 'Chip cache hit', cpuCacheHit: 'CPU cache hit', + combinedCacheHit: 'Combined chip + CPU cache hit', enabledLegacy: 'Enabled (legacy data)', disabledLegacy: 'Disabled (legacy data)', none: 'None', @@ -40,6 +42,7 @@ export const POINT_SUMMARY_STRINGS = { concurrency: '并发数', gpuCacheHit: '芯片 cache 命中率', cpuCacheHit: 'CPU cache 命中率', + combinedCacheHit: '芯片 + CPU 综合 cache 命中率', enabledLegacy: '已启用（旧版数据）', disabledLegacy: '已禁用（旧版数据）', none: '无', @@ -71,6 +74,7 @@ export function PointSummary({ meta }: { meta: PointMeta }) { const locale = useLocale(); const t = POINT_SUMMARY_STRINGS[locale]; const showCpuCacheHit = isKvOffloadEnabled(meta); + const showCombinedCacheHit = showCpuCacheHit && frameworkFamily(meta.framework) === 'trt'; const offloadBackend = versionedComponentLabel( meta.kv_offload_backend, meta.kv_offload_backend_version, @@ -104,8 +108,11 @@ export function PointSummary({ meta }: { meta: PointMeta }) { {transferEngine && } {router && } - - {showCpuCacheHit && ( + + {showCpuCacheHit && !showCombinedCacheHit && ( )} {meta.isl !== null && } diff --git a/packages/app/src/components/inference/agentic-point/time-series-chart.tsx b/packages/app/src/components/inference/agentic-point/time-series-chart.tsx index 1e5a4166c..1c4291df3 100644 --- a/packages/app/src/components/inference/agentic-point/time-series-chart.tsx +++ b/packages/app/src/components/inference/agentic-point/time-series-chart.tsx @@ -5,7 +5,16 @@ import { useMemo } from 'react'; import type { TimeSeriesPoint } from '@/hooks/api/use-trace-server-metrics'; import { ChartHover, type HoverItem } from './chart-hover'; -import { CHART_PAD, ChartEmpty, fmtCount, fmtSeconds } from './chart-shared'; +import { + CHART_PAD, + ChartEmpty, + ChartLegend, + LEGEND_BASELINE_OFFSET, + fmtCount, + fmtSeconds, + type ChartLegendEntry, +} from './chart-shared'; +import { layoutChartLegend } from './chart-legend'; import { interpAt, maxTimeSeriesValue, type ChartSeries } from './time-series-math'; import { useLocale } from '@/lib/use-locale'; @@ -53,11 +62,35 @@ export function TimeSeriesChart({ }: TimeSeriesChartProps) { const locale = useLocale(); const W = width; - const H = height; + + // Legend entries, skipping series flagged hideFromHover so per-engine + // underlays don't take a legend slot with no visible line. + const legendEntries = useMemo( + () => + series + .filter((s) => !s.hideFromHover) + .map((s) => ({ label: s.name, color: s.color, strokeWidth: s.strokeWidth ?? 2 })), + [series], + ); + + // Wrap the legend to as many rows as the labels need, then grow the viewBox + // and the bottom padding by the same amount. The plot area keeps its exact + // geometry and the extra rows extend the SVG downward. + const legend = useMemo( + () => + layoutChartLegend( + legendEntries.map((e) => e.label), + W - PAD.left - PAD.right, + ), + [legendEntries, W], + ); + const H = height + legend.extraHeight; + const padBottom = PAD.bottom + legend.extraHeight; + const pad = useMemo(() => ({ ...PAD, bottom: padBottom }), [padBottom]); const layout = useMemo(() => { const innerW = W - PAD.left - PAD.right; - const innerH = H - PAD.top - PAD.bottom; + const innerH = H - PAD.top - padBottom; const xMax = Math.max(durationS, 1); // Fold reference-line values into the auto max so a ceiling above the data // (e.g. KV-cache pool >> working set) still renders inside the plot. @@ -67,7 +100,7 @@ export function TimeSeriesChart({ const xScale = (t: number) => PAD.left + (t / xMax) * innerW; const yScale = (v: number) => PAD.top + (1 - v / yMax) * innerH; return { innerW, innerH, xMax, yMax, xScale, yScale }; - }, [series, durationS, yMaxOpt, refLines, W, H]); + }, [series, durationS, yMaxOpt, refLines, W, H, padBottom]); const { innerW, innerH, xMax, yMax, xScale, yScale } = layout; @@ -99,7 +132,7 @@ export function TimeSeriesChart({ } return ( - + {/* y-axis gridlines + labels */} {yTickVals.map((v, i) => { const y = yScale(v); @@ -225,7 +258,7 @@ export function TimeSeriesChart({ })} {yAxisLabel} )} - {/* Legend — skip series flagged hideFromHover so per-engine - underlays don't clutter the chip row. */} - {(() => { - const visible = series.filter((s) => !s.hideFromHover); - const chipY = H - 8; - const chipW = innerW / Math.max(1, visible.length); - return visible.map((s, i) => { - const x = PAD.left + i * chipW; - return ( - - - - {s.name} - - - ); - }); - })()} + ); } @@ -333,7 +347,6 @@ export function StackedAreaChart({ const locale = useLocale(); const sourceLabels = SOURCE_LABELS[locale]; const W = width; - const H = height; const computed = useMemo(() => { const entries = Object.entries(sourceSeries).filter(([, v]) => v.length > 0); @@ -397,8 +410,24 @@ export function StackedAreaChart({ } const colorFor = (name: string): string => colorByName.get(name) ?? FALLBACK_PALETTE[0]!; + // Wrap the legend, then grow the viewBox and bottom padding by the same + // amount so the plot area is untouched (see TimeSeriesChart). + const legendEntries: ChartLegendEntry[] = stackOrder.map((name) => ({ + label: sourceLabels[name] ?? name, + color: colorFor(name), + swatch: 'area', + opacity: 0.75, + })); + const legend = layoutChartLegend( + legendEntries.map((e) => e.label), + W - PAD.left - PAD.right, + ); + const H = height + legend.extraHeight; + const padBottom = PAD.bottom + legend.extraHeight; + const pad = { ...PAD, bottom: padBottom }; + const innerW = W - PAD.left - PAD.right; - const innerH = H - PAD.top - PAD.bottom; + const innerH = H - PAD.top - padBottom; const xMax = Math.max(durationS, 1); const xScale = (t: number) => PAD.left + (t / xMax) * innerW; const yScale = (v: number) => PAD.top + (1 - v) * innerH; @@ -443,7 +472,7 @@ export function StackedAreaChart({ const yTickVals = [0, 0.25, 0.5, 0.75, 1]; return ( - + {yTickVals.map((v, i) => { const y = yScale(v); return ( @@ -499,7 +528,7 @@ export function StackedAreaChart({ })} {locale === 'zh' ? '预填充 token 占比' : '% of prefill tokens'} - {(() => { - const chipY = H - 8; - const chipW = innerW / Math.max(1, layers.length); - return layers.map((l, i) => { - const x = PAD.left + i * chipW; - return ( - - - - {sourceLabels[l.name] ?? l.name} - - - ); - }); - })()} + ); } diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index 1f33e02b1..317375e46 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -500,6 +500,23 @@ describe('generateTooltipContent', () => { expect(enabled).toContain('CPU Cache Hit Rate: 42.0%'); }); + it('labels TRTLLM offload cache hits as a combined chip and CPU rate', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ + framework: 'dynamo-trt', + kv_offloading: 'dram', + server_gpu_cache_hit_rate: 0.978, + server_cpu_cache_hit_rate: undefined, + }), + }), + ); + + expect(html).toContain('Combined Chip + CPU Cache Hit Rate: 97.8%'); + expect(html).not.toContain('CPU Cache Hit Rate:'); + expect(html).not.toContain('Chip Cache Hit Rate:'); + }); + it('uses Chinese labels for new cache metadata on /zh surfaces', () => { const html = generateTooltipContent( tooltipConfig({ diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 568fa84ce..aabacaff1 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -2,6 +2,7 @@ import { formatNumber, getDisplayLabel } from '@/lib/utils'; import { specMethodDisplayLabel } from '@/lib/compare-variant-slug'; import { agenticDetailHref } from '@/lib/agentic-detail-link'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; +import { frameworkFamily } from '@/lib/framework-family'; import type { Locale } from '@/lib/i18n'; import { isKvOffloadEnabled } from '@/lib/kv-offload'; @@ -185,6 +186,7 @@ const CACHE_STRINGS = { router: 'Router', gpuHitRate: 'Chip Cache Hit Rate', cpuHitRate: 'CPU Cache Hit Rate', + combinedHitRate: 'Combined Chip + CPU Cache Hit Rate', theoreticalHitRate: 'Theoretical Cache Hit Rate', legacyEnabled: 'Enabled (legacy data)', legacyDisabled: 'Disabled (legacy data)', @@ -196,6 +198,7 @@ const CACHE_STRINGS = { router: '路由器', gpuHitRate: '芯片 Cache 命中率', cpuHitRate: 'CPU Cache 命中率', + combinedHitRate: '芯片 + CPU 综合 Cache 命中率', theoreticalHitRate: '理论 Cache 命中率', legacyEnabled: '已启用（旧版数据）', legacyDisabled: '已禁用（旧版数据）', @@ -234,8 +237,10 @@ const generateCacheMetadataHTML = (d: InferenceData, locale: Locale): string => const gpuHit = formatPct(d.server_gpu_cache_hit_rate); const cpuHit = formatPct(d.server_cpu_cache_hit_rate); const theoreticalHit = formatPct(d.theoretical_cache_hit_rate); - if (gpuHit) parts.push(tooltipLine(t.gpuHitRate, gpuHit)); - if (cpuHit && isKvOffloadEnabled(d)) parts.push(tooltipLine(t.cpuHitRate, cpuHit)); + const offloadEnabled = isKvOffloadEnabled(d); + const combinedHit = offloadEnabled && frameworkFamily(d.framework) === 'trt'; + if (gpuHit) parts.push(tooltipLine(combinedHit ? t.combinedHitRate : t.gpuHitRate, gpuHit)); + if (cpuHit && offloadEnabled && !combinedHit) parts.push(tooltipLine(t.cpuHitRate, cpuHit)); if (theoreticalHit) parts.push(tooltipLine(t.theoreticalHitRate, theoreticalHit)); return parts.join(''); }; diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index bc3ddddf8..116556aeb 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -715,7 +715,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/agentic-aggregates.ts', - sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18', + sourceSha256: 'b8b72a37ca7a67a1f234036fcbd9e3edacbd7031e0a68fb793944fc4de7030da', reviewArea: { en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.', zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。', diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index c1ce8f2f0..f71d7839a 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -18,6 +18,7 @@ * [--limit N] only process the first N candidate rows (useful for * smoke-tests on a fresh deploy) * [--force] recompute every row, even if version already matches + * [--run-id N] only process trace rows linked to one GitHub workflow run * [--yes] skip the confirmation prompt */ @@ -33,11 +34,13 @@ import { createAdminSql } from './etl/db-utils.js'; import { jsonbParam, parseLimitForceFlags, + parseRunIdFlag, runBackfillMain, runCandidateIdBackfill, } from './lib/backfill-runner.js'; const flags = parseLimitForceFlags(); +const githubRunId = parseRunIdFlag(); // Neon's HTTP response and JS drivers should not receive a 100+ MB bytea as // one value. Slice oversized TOAST values into independent bounded reads and @@ -69,9 +72,21 @@ async function main(): Promise { console.log(` STATS_VERSION = ${STATS_VERSION}`); console.log(` force = ${flags.force}`); console.log(` limit = ${flags.limit ?? 'none'}`); + console.log(` run_id = ${githubRunId ?? 'all'}`); await runCandidateIdBackfill( async () => { + const runFilter = githubRunId + ? sql` + and exists ( + select 1 + from benchmark_results br + join latest_workflow_runs wr on wr.id = br.workflow_run_id + where br.trace_replay_id = agentic_trace_replay.id + and wr.github_run_id = ${githubRunId} + ) + ` + : sql``; // Find candidates: rows missing stats, or whose stored version is stale. // Using >>'version'::int comparison would error on null; coalesce to -1 so // null-stats rows always count as stale. @@ -79,14 +94,16 @@ async function main(): Promise { ? await sql<{ id: number }[]>` select id from agentic_trace_replay + where true ${runFilter} order by id ${flags.limit ? sql`limit ${flags.limit}` : sql``} ` : await sql<{ id: number }[]>` select id from agentic_trace_replay - where aggregate_stats is null - or coalesce((aggregate_stats->>'version')::int, -1) <> ${STATS_VERSION} + where (aggregate_stats is null + or coalesce((aggregate_stats->>'version')::int, -1) <> ${STATS_VERSION}) + ${runFilter} order by id ${flags.limit ? sql`limit ${flags.limit}` : sql``} `; @@ -132,7 +149,12 @@ async function main(): Promise { // fields haven't changed since v3), so skip re-reading the huge server // blob and carry its KV/prefix distributions forward. const storedVersion = row.aggregate_stats?.version; - if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { + if ( + !flags.force && + storedVersion !== undefined && + storedVersion >= 3 && + storedVersion < STATS_VERSION + ) { stats = mergeProfileStatsUpgrade(row.aggregate_stats!, profileStats); } else { const [serverRow] = await sql<{ server_metrics_json_gz: Buffer | null }[]>` diff --git a/packages/db/src/backfill-chart-series.ts b/packages/db/src/backfill-chart-series.ts index 800499534..12a04ae4d 100644 --- a/packages/db/src/backfill-chart-series.ts +++ b/packages/db/src/backfill-chart-series.ts @@ -21,6 +21,7 @@ * [--force] recompute every row, even if version already matches * [--shard-count N] split work across N independent processes * [--shard-index N] zero-based shard handled by this process + * [--run-id N] only process trace rows linked to one GitHub workflow run * [--yes] skip the confirmation prompt */ @@ -30,11 +31,13 @@ import { createAdminSql } from './etl/db-utils.js'; import { jsonbParam, parseLimitForceFlags, + parseRunIdFlag, runBackfillMain, runCandidateIdBackfill, } from './lib/backfill-runner.js'; const flags = parseLimitForceFlags(); +const githubRunId = parseRunIdFlag(); const sql = createAdminSql({ noSsl: hasNoSslFlag(), @@ -48,6 +51,7 @@ async function main(): Promise { console.log(` force = ${flags.force}`); console.log(` limit = ${flags.limit ?? 'none'}`); console.log(` shard = ${flags.shardIndex + 1}/${flags.shardCount}`); + console.log(` run_id = ${githubRunId ?? 'all'}`); await runCandidateIdBackfill( async () => { @@ -56,12 +60,24 @@ async function main(): Promise { // null and the API serves them via the slow path (which also returns // null because there's no blob to parse — so the page falls into the // "no stored trace_replay blob" branch). + const runFilter = githubRunId + ? sql` + and exists ( + select 1 + from benchmark_results br + join latest_workflow_runs wr on wr.id = br.workflow_run_id + where br.trace_replay_id = agentic_trace_replay.id + and wr.github_run_id = ${githubRunId} + ) + ` + : sql``; const candidates = flags.force ? await sql<{ id: number }[]>` select id from agentic_trace_replay where server_metrics_json_gz is not null and mod(id, ${flags.shardCount}) = ${flags.shardIndex} + ${runFilter} -- Restore the newest, most actively viewed runs first. The backfill is -- idempotent, so an interrupted pass resumes with only stale rows. order by id desc @@ -76,6 +92,7 @@ async function main(): Promise { chart_series is null or coalesce((chart_series->>'version')::int, -1) <> ${CHART_SERIES_VERSION} ) + ${runFilter} -- Restore the newest, most actively viewed runs first. The backfill is -- idempotent, so an interrupted pass resumes with only stale rows. order by id desc diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 07729b59b..85cd7c1ac 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -193,6 +193,12 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([ 'vllm:prefix_cache_queries', 'vllm:gpu_prefix_cache_hits', 'vllm:gpu_prefix_cache_queries', + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ]); /** diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 54295cc77..312853d24 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -146,6 +146,20 @@ function kvBlob(profiling: unknown[], warmup: unknown[] = []) { ); } +function buildTrtllmSeries( + endpoint_url: string, + dynamo_component: 'prefill' | 'backend', + value: number, + field: 'rate' | 'avg' | 'sum', + durationNs = 1e9, +) { + return { + endpoint_url, + labels: { dynamo_component, worker_id: `${dynamo_component}-worker` }, + timeslices: [{ start_ns: 0, end_ns: durationNs, [field]: value }], + }; +} + describe('computeChartSeries', () => { it('returns null when the blob is null', async () => { expect(await computeChartSeries(null)).toBeNull(); @@ -708,6 +722,125 @@ describe('computeChartSeries', () => { expect(result?.metricSources).toEqual([]); }); + + it('extracts native TensorRT-LLM metrics and preserves disaggregated worker roles', async () => { + const prefillUrl = 'http://prefill-a.internal.test:7500/metrics'; + const decodeUrl = 'http://decode-a.internal.test:7501/metrics'; + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 0.3, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 0.7, 'avg'), + ], + }, + trtllm_kv_cache_host_utilization: { + series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], + }, + trtllm_prefill_batch_tokens: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 30, 'sum', 0.5e9), + buildTrtllmSeries(decodeUrl, 'backend', 60, 'sum', 0.5e9), + ], + }, + trtllm_prompt_cached_tokens: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'), + buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'), + ], + }, + trtllm_generation_tokens: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')], + }, + trtllm_num_requests_running: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 2, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 3, 'avg'), + ], + }, + trtllm_num_requests_waiting: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 4, 'avg')], + }, + }, + }); + + const result = await computeChartSeries(gzipSync(Buffer.from(json)), { + framework: 'trtllm', + disagg: true, + }); + + expect(result?.kvCacheUsage).toEqual([{ t: 0, value: 0.5 }]); + expect(result?.hostKvCacheUsage).toEqual([{ t: 0, value: 0.25 }]); + expect(result?.prefixCacheHitRate).toEqual([{ t: 0, value: 0.4 }]); + expect(result?.queueDepth).toEqual([{ t: 0, running: 5, waiting: 4, total: 9 }]); + expect(result?.prefillTps).toEqual([{ t: 0, value: 300 }]); + expect(result?.decodeTps).toEqual([{ t: 0, value: 50 }]); + expect(result?.promptTokensBySource).toEqual({ + 'cache hit (HBM)': [{ t: 0, value: 120 }], + 'compute (miss)': [{ t: 0, value: 180 }], + }); + expect(result?.metricSources.map(({ source }) => [source.role, source.endpointUrl])).toEqual([ + ['prefill', prefillUrl], + ['decode', decodeUrl], + ]); + expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]); + expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]); + }); + + it('keeps Dynamo TRT ranks within each worker and falls back to native-only endpoints', async () => { + const prefillUrls = ['http://prefill-a:7500/metrics', 'http://prefill-b:7500/metrics']; + const decodeUrl = 'http://decode:7500/metrics'; + const result = await computeChartSeries( + gzipSync( + Buffer.from( + JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + ...prefillUrls.map((url) => buildTrtllmSeries(url, 'prefill', 0.9, 'avg')), + buildTrtllmSeries(decodeUrl, 'backend', 0.8, 'avg'), + ], + }, + dynamo_component_gpu_cache_usage_percent: { + series: prefillUrls.flatMap((endpoint_url) => + [0, 1].map((rank) => ({ + endpoint_url, + labels: { + dynamo_component: 'prefill', + model: 'GLM-5.2-NVFP4', + dp_rank: String(rank), + }, + timeslices: [{ start_ns: 0, end_ns: 1e9, avg: rank === 0 ? 0.2 : 0.6 }], + })), + ), + }, + trtllm_num_requests_running: { + series: prefillUrls.map((url) => buildTrtllmSeries(url, 'prefill', 2, 'avg')), + }, + }, + }), + ), + ), + { framework: 'dynamo-trt', disagg: true }, + ); + + expect(result?.metricSources).toHaveLength(3); + expect(result?.kvCacheUsageByEngine).toHaveLength(5); + expect(result?.kvCacheUsage[0]?.value).toBeCloseTo(0.48); + for (const url of prefillUrls) { + const source = result?.metricSources.find((s) => s.source.endpointUrl === url); + expect(source?.source.dpRank).toBeNull(); + expect(source?.kvCacheUsage).toEqual([{ t: 0, value: 0.4 }]); + expect(source?.kvCacheUsageByEngine).toEqual([ + { engineLabel: '0', points: [{ t: 0, value: 0.2 }] }, + { engineLabel: '1', points: [{ t: 0, value: 0.6 }] }, + ]); + expect(source?.queueDepth).toEqual([{ t: 0, running: 2, waiting: 0, total: 2 }]); + } + expect(result?.metricSources.find((s) => s.source.role === 'decode')?.kvCacheUsage).toEqual([ + { t: 0, value: 0.8 }, + ]); + }); }); // ── Summed series on the canonical grid (v14) ─────────────────────────── diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index a00d81a19..da321b78b 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -120,8 +120,17 @@ import { * per-rank detail stays reachable as that source's own * `kvCacheUsageByEngine`. * + * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV + * metrics, including per-prefill/decode source series for Dynamo disaggregation. + * + * v17: derive TensorRT-LLM prompt throughput from its actual metric shape: + * cached-token counter rate plus the prefill-batch-token histogram sum per + * timeslice. TRT-LLM does not currently expose the prompt-token counter that + * v16 expected. + * + * v18: retain Dynamo's per-DP-rank KV gauges within each TRT-LLM worker. */ -export const CHART_SERIES_VERSION = 15; +export const CHART_SERIES_VERSION = 18; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -204,8 +213,11 @@ interface RawSlice { end_ns?: number; avg?: number; rate?: number; + sum?: number; } +type RawSliceField = 'avg' | 'rate' | 'sumRate'; + interface RawSeries { endpoint_url?: string; labels?: Record; @@ -244,6 +256,20 @@ export const CHART_METRIC_KEYS = new Set([ 'sglang:realtime_tokens', 'sglang:hicache_host_used_tokens', 'sglang:hicache_host_total_tokens', + // TensorRT-LLM + 'dynamo_component_gpu_cache_usage_percent', + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_host_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', + 'trtllm_prefill_batch_tokens', + 'trtllm_generation_tokens', + 'trtllm_generation_tokens_total', + 'trtllm_num_requests_running', + 'trtllm_num_requests_waiting', ]); /** @@ -473,7 +499,7 @@ const MIRROR_RATE_RELATIVE_TOLERANCE = 0.05; * request counts) whose mirrors agree almost exactly. Rates need a relative * test because their magnitude is unbounded. */ -function looksMirrored(means: readonly number[], field: 'avg' | 'rate'): boolean { +function looksMirrored(means: readonly number[], field: RawSliceField): boolean { const spread = Math.max(...means) - Math.min(...means); if (field === 'avg') return spread <= MIRROR_MEAN_TOLERANCE; const scale = Math.max(...means.map((m) => Math.abs(m))); @@ -555,7 +581,7 @@ function spanOf(scrapes: ScrapeMap): number { function resolveComponents( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate' = 'avg', + field: RawSliceField = 'avg', ): ResolvedEngine[] { const groups = new Map(); for (const s of series ?? []) { @@ -573,7 +599,16 @@ function resolveComponents( } for (const ts of s.timeslices ?? []) { if (typeof ts.start_ns !== 'number' || !Number.isFinite(ts.start_ns)) continue; - const value = ts[field]; + const durationS = + typeof ts.start_ns === 'number' && typeof ts.end_ns === 'number' + ? (ts.end_ns - ts.start_ns) / 1e9 + : 0; + const value = + field === 'sumRate' + ? typeof ts.sum === 'number' && durationS > 0 + ? ts.sum / durationS + : undefined + : ts[field]; if (typeof value !== 'number' || !Number.isFinite(value)) continue; const at = scrapes.get(ts.start_ns); if (at) { @@ -915,7 +950,7 @@ function sumOntoGrid( function summedSeries( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate', + field: RawSliceField, tickS: number | null, ): TimeSeriesPoint[] { const components = resolveComponents(series, tOf, field).map((c) => c.points); @@ -972,11 +1007,24 @@ function buildSeriesFromMetrics( // KV cache usage (gauge, 0..1) — average across engines so the value // stays a fraction (each engine has its own KV pool). - const kvSeries = pickSeries( - 'vllm:kv_cache_usage_perc', - 'vllm:gpu_cache_usage_perc', - 'sglang:token_usage', - ); + const nativeTrtKv = metrics['trtllm_kv_cache_utilization']?.series ?? []; + const dynamoTrtKv = + selectServerMetricsAdapter(context).id === 'trtllm' + ? (metrics['dynamo_component_gpu_cache_usage_percent']?.series ?? []) + : []; + const rankedEndpoints = new Set(dynamoTrtKv.map((s) => s.endpoint_url)); + // Dynamo gauges omit worker_id. Preserve endpoint identity so two workers + // with similar utilization and the same DP rank cannot be mistaken for mirrors. + const trtKv = [ + ...dynamoTrtKv.map((s) => ({ + ...s, + labels: { ...s.labels, metric_endpoint: s.endpoint_url ?? '' }, + })), + ...nativeTrtKv.filter((s) => !rankedEndpoints.has(s.endpoint_url)), + ]; + const kvSeries = + pickSeries('vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage') ?? + trtKv; // One entry per logical engine (v13) — mirrored API-server frontends and the // warmup/profiling phase split are collapsed here rather than showing up as // extra "engines". @@ -989,11 +1037,18 @@ function buildSeriesFromMetrics( // Prefix cache hit rate per scrape: Σhits.rate / Σqueries.rate across // engines, joined on start_ns. SGLang names: cached_tokens / prompt_tokens. - const hitsSeries = pickSeries('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const hitsSeries = pickSeries( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); const qsSeries = pickSeries( 'vllm:prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS); const qsOnGrid = summedSeries(qsSeries, tOf, 'rate', tickS); @@ -1003,10 +1058,25 @@ function buildSeriesFromMetrics( const q = qsByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q }); } + if (prefixCacheHitRate.length === 0) { + prefixCacheHitRate.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_hit_rate']?.series, tOf), + ), + ); + } // Queue depth: sum running + waiting across engines per timeslice. - const runSeries = pickSeries('vllm:num_requests_running', 'sglang:num_running_reqs'); - const waitSeries = pickSeries('vllm:num_requests_waiting', 'sglang:num_queue_reqs'); + const runSeries = pickSeries( + 'vllm:num_requests_running', + 'sglang:num_running_reqs', + 'trtllm_num_requests_running', + ); + const waitSeries = pickSeries( + 'vllm:num_requests_waiting', + 'sglang:num_queue_reqs', + 'trtllm_num_requests_waiting', + ); const runOnGrid = summedSeries(runSeries, tOf, 'avg', tickS); const waitByT = byT(summedSeries(waitSeries, tOf, 'avg', tickS)); const runByT = byT(runOnGrid); @@ -1025,11 +1095,45 @@ function buildSeriesFromMetrics( // work. const counterRate = (...names: string[]): TimeSeriesPoint[] => summedSeries(pickSeries(...names), tOf, 'rate', tickS); - const prefillTps = counterRate('vllm:prompt_tokens', 'sglang:prompt_tokens'); - const decodeTps = counterRate('vllm:generation_tokens', 'sglang:generation_tokens'); + const promptCounterTps = counterRate( + 'vllm:prompt_tokens', + 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', + ); + const decodeTps = counterRate( + 'vllm:generation_tokens', + 'sglang:generation_tokens', + 'trtllm_generation_tokens', + 'trtllm_generation_tokens_total', + ); // Tokens served from prefix cache per scrape. Lets the frontend derive // "cumulative unique input tokens served" = cumsum(prefillTps) − cumsum(hits). - const prefixCacheHitsTps = counterRate('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const prefixCacheHitsTps = counterRate( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); + const trtllmComputedPromptTps = summedSeries( + metrics['trtllm_prefill_batch_tokens']?.series, + tOf, + 'sumRate', + tickS, + ); + const prefillTps = + promptCounterTps.length > 0 + ? promptCounterTps + : sumOntoGrid([trtllmComputedPromptTps, prefixCacheHitsTps], tickS); + if (prefixCacheHitRate.length === 0 && (!qsSeries || qsSeries.length === 0)) { + const prefillByT = byT(prefillTps); + for (const { t, value: cached } of prefixCacheHitsTps) { + const prompt = prefillByT.get(t); + if (prompt !== undefined && prompt > 0) { + prefixCacheHitRate.push({ t, value: cached / prompt }); + } + } + } // SGLang hicache: host-pool KV cache utilization as used/total per // timeslice. Both metrics are gauges in absolute tokens. Total stays @@ -1049,6 +1153,13 @@ function buildSeriesFromMetrics( hostKvCacheUsage.push({ t, value: used / total }); } } + if (hostKvCacheUsage.length === 0) { + hostKvCacheUsage.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_host_utilization']?.series, tOf), + ), + ); + } // Per-source prompt tokens — sum across engines per source label. // vllm: vllm:prompt_tokens_by_source has one series per source label @@ -1115,14 +1226,42 @@ function buildSeriesFromMetrics( const arr = summedSeries(seriesForSource, tOf, 'rate', tickS).filter((p) => p.value > 0); if (arr.length > 0) promptTokensBySource[source] = arr; } + if (Object.keys(promptTokensBySource).length === 0) { + const cached = prefixCacheHitsTps.filter((point) => point.value > 0); + const cachedByT = byT(prefixCacheHitsTps); + const computed = + trtllmComputedPromptTps.length > 0 + ? trtllmComputedPromptTps.filter((point) => point.value > 0) + : promptCounterTps + .map(({ t, value: prompt }) => ({ + t, + value: Math.max(0, prompt - (cachedByT.get(t) ?? 0)), + })) + .filter((point) => point.value > 0); + if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached; + if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed; + } const metricSources: MetricSourceSeries[] = []; const adapter = selectServerMetricsAdapter(context); if (includeMetricSources && context.disagg && adapter.id !== 'generic') { + const endpointRoles = new Map(); + for (const metric of Object.values(metrics)) { + for (const series of metric.series ?? []) { + const endpointUrl = series.endpoint_url; + const role = series.labels?.['disaggregation_mode'] ?? series.labels?.['dynamo_component']; + if (endpointUrl && role) endpointRoles.set(endpointUrl, role); + } + } const grouped = new Map(); for (const [metricName, metric] of Object.entries(metrics)) { for (const series of metric.series ?? []) { - const source = adapter.identifySource(series); + const roleHint = series.endpoint_url ? endpointRoles.get(series.endpoint_url) : undefined; + const identifiedSeries = + roleHint && !series.labels?.['disaggregation_mode'] + ? { ...series, labels: { ...series.labels, disaggregation_mode: roleHint } } + : series; + const source = adapter.identifySource(identifiedSeries); let group = grouped.get(source.id); if (!group) { group = { source, metrics: {} }; diff --git a/packages/db/src/etl/server-metrics-adapters.ts b/packages/db/src/etl/server-metrics-adapters.ts index 5e80e8bd6..4807ce002 100644 --- a/packages/db/src/etl/server-metrics-adapters.ts +++ b/packages/db/src/etl/server-metrics-adapters.ts @@ -78,6 +78,36 @@ const dynamoAdapter: ServerMetricsAdapter = { }, }; +const trtllmAdapter: ServerMetricsAdapter = { + id: 'trtllm', + matches: ({ framework }) => framework?.toLowerCase().includes('trt') ?? false, + identifySource(series) { + const labels = series.labels ?? {}; + const nativeRole = labels['disaggregation_mode'] ?? labels['dynamo_component'] ?? null; + const role: MetricSourceRole = + nativeRole === 'prefill' + ? 'prefill' + : nativeRole === 'decode' || nativeRole === 'backend' + ? 'decode' + : nativeRole === 'aggregated' + ? 'combined' + : 'unknown'; + const endpointUrl = series.endpoint_url ?? null; + const workerId = labels['worker_id'] ?? null; + return { + // Native TRT metrics and Dynamo's rank-labelled gauges share one endpoint. + id: stableId('trtllm', [role, endpointUrl ?? workerId]), + adapter: 'trtllm', + role, + endpointUrl, + nativeRole, + workerId, + dpRank: null, + engine: null, + }; + }, +}; + const genericAdapter: ServerMetricsAdapter = { id: 'generic', matches: () => true, @@ -100,7 +130,7 @@ const genericAdapter: ServerMetricsAdapter = { }, }; -const ADAPTERS: readonly ServerMetricsAdapter[] = [dynamoAdapter, genericAdapter]; +const ADAPTERS: readonly ServerMetricsAdapter[] = [trtllmAdapter, dynamoAdapter, genericAdapter]; export function selectServerMetricsAdapter(context: ServerMetricsContext): ServerMetricsAdapter { return ADAPTERS.find((adapter) => adapter.matches(context)) ?? genericAdapter; diff --git a/packages/db/src/lib/backfill-runner.test.ts b/packages/db/src/lib/backfill-runner.test.ts index cf6b40f9c..24cbe5fe4 100644 --- a/packages/db/src/lib/backfill-runner.test.ts +++ b/packages/db/src/lib/backfill-runner.test.ts @@ -2,10 +2,32 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { parseLimitForceFlags, + parseRunIdFlag, runCandidateIdBackfill, runPerIdBackfill, } from './backfill-runner.js'; +describe('parseRunIdFlag', () => { + it('leaves unscoped backfills unchanged', () => { + expect(parseRunIdFlag(['bun', 'backfill.ts', '--force'])).toBeUndefined(); + }); + + it('preserves large GitHub run IDs', () => { + expect(parseRunIdFlag(['bun', 'backfill.ts', '--run-id', '33418433573', '--yes'])).toBe( + 33418433573, + ); + }); + + it.each([undefined, '', '--yes', 'all', '0', '-1', '1.5', '1e3', '9007199254740992'])( + 'rejects an invalid run selector: %s', + (value) => { + const argv = ['bun', 'backfill.ts', '--run-id']; + if (value !== undefined) argv.push(value); + expect(() => parseRunIdFlag(argv)).toThrow('--run-id requires a positive integer'); + }, + ); +}); + describe('parseLimitForceFlags', () => { const originalArgv = process.argv; afterEach(() => { diff --git a/packages/db/src/lib/backfill-runner.ts b/packages/db/src/lib/backfill-runner.ts index 5fde13ede..fd92b8a1d 100644 --- a/packages/db/src/lib/backfill-runner.ts +++ b/packages/db/src/lib/backfill-runner.ts @@ -16,6 +16,18 @@ export interface LimitForceFlags { shardIndex: number; } +/** Restrict a backfill to one GitHub run; malformed selectors must never scan all rows. */ +export function parseRunIdFlag(argv: readonly string[] = process.argv): number | undefined { + const index = argv.indexOf('--run-id'); + if (index === -1) return undefined; + const raw = argv[index + 1]; + const runId = Number(raw); + if (!raw || !/^\d+$/u.test(raw) || !Number.isSafeInteger(runId) || runId < 1) { + throw new Error('--run-id requires a positive integer GitHub workflow run ID'); + } + return runId; +} + /** Parse the standard `--limit N` / `--force` backfill flags from argv. */ export function parseLimitForceFlags(): LimitForceFlags { let limit: number | null = null; diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 89ec4b914..945b515a4 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,6 +129,47 @@ describe('extractServerMetricSamples', () => { expect(out.kvCacheUtil).toEqual([]); expect(out.prefixCacheHitRate).toEqual([]); }); + + it('extracts TensorRT-LLM KV utilization and prefix cache hit rate', () => { + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + { + timeslices: [ + { start_ns: 0, avg: 0.2 }, + { start_ns: 1, avg: 0.6 }, + ], + }, + ], + }, + trtllm_prompt_cached_tokens: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 70 }, + { start_ns: 1, rate: 20 }, + ], + }, + ], + }, + trtllm_prompt_tokens: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 100 }, + { start_ns: 1, rate: 50 }, + ], + }, + ], + }, + }, + }); + + const out = extractServerMetricSamples(json); + expect(out.kvCacheUtil).toEqual([0.2, 0.6]); + expect(out.prefixCacheHitRate).toEqual([0.7, 0.4]); + }); }); /** The write-back payload as bound to the UPDATE (a partial aggregate_stats). */ diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 9511d2484..aecb32611 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -157,6 +157,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); const kvCacheUtil = [...aggregateSeriesByStart(kvSeriesAll, 'avg', 'avg').values()]; @@ -167,6 +168,8 @@ export function extractServerMetricSamples(json: string): { 'vllm:prefix_cache_hits', 'vllm:gpu_prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', ); const queriesAll = pickFirstNonEmpty( metrics, @@ -174,6 +177,8 @@ export function extractServerMetricSamples(json: string): { 'vllm:gpu_prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsByT = aggregateSeriesByStart(hitsAll, 'rate', 'sum'); const qByT = aggregateSeriesByStart(queriesAll, 'rate', 'sum'); @@ -182,6 +187,14 @@ export function extractServerMetricSamples(json: string): { const q = qByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push(h / q); } + if (prefixCacheHitRate.length === 0) { + const directRate = aggregateSeriesByStart( + metrics['trtllm_kv_cache_hit_rate']?.series ?? [], + 'avg', + 'avg', + ); + prefixCacheHitRate.push(...directRate.values()); + } return { kvCacheUtil, prefixCacheHitRate }; } @@ -200,6 +213,13 @@ const TARGET_METRIC_KEYS = new Set([ 'sglang:token_usage', 'sglang:cached_tokens', 'sglang:prompt_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ]); /**