Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions console/web/src/pages/TracesV2/hooks/useAllSpans.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* are pruned as frames arrive, and a hard cap keeps a busy engine from
* growing the map without limit (oldest effective-end evicted first).
* Paused / hidden-tab frames are dropped, matching the rows stream; a
* reconnect or unpause re-seeds once, REPLACING the map — the engine store
* is the source of truth, and merging would immortalize a stale pending
* span whose close frame was lost across an engine restart.
* reconnect, unpause, or tab-visible re-seeds once, REPLACING the map —
* the engine store is the source of truth, and merging would immortalize
* a stale pending span whose close frame was lost across an engine
* restart.
*
* Engines without the all-spans stream simply never deliver a frame: the
* strip then shows the seed's spans and refreshes on reconnects only.
Expand Down Expand Up @@ -144,9 +145,24 @@ export function useAllSpans(isPaused: boolean): readonly StoredSpan[] {
void seedRef.current()
}
})
// Hidden-tab frames are dropped above, so the map has a hole after a
// tab switch — re-seed on return, mirroring the list's recovery in
// `useTraceData`. (REPLACE semantics, see the module docstring.)
let offVisibility: (() => void) | undefined
if (typeof document !== 'undefined') {
const onVisible = () => {
if (document.visibilityState === 'visible' && !isPausedRef.current) {
void seedRef.current()
}
}
document.addEventListener('visibilitychange', onVisible)
offVisibility = () =>
document.removeEventListener('visibilitychange', onVisible)
}
stop = () => {
offFeed()
offConn()
offVisibility?.()
}
})()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { StoredSpan } from '../api/traces'
import type { TimelineSpan } from '../components/timeline/layout'
import type { SpanFilterSelection } from '../lib/spanFilters'
import {
liveTraceIds,
mergeFetchedVerdicts,
reconcileTraceVisibility,
rowRootFilterKeys,
Expand Down Expand Up @@ -208,6 +209,50 @@ describe('reconcileTraceVisibility', () => {
).toEqual(['t-1'])
})

it('keeps a visible verdict when the visible bars prune out but hidden ones linger', () => {
// The mid-run vanish: the feed retains ~2min, so during a long quiet
// stretch a turn's visible bars prune away while its hidden
// bookkeeping keeps the trace in the feed. Visibility is monotone —
// the row must not flip back to hidden.
const verdicts = new Map<string, boolean>()
const sel = selection({ hiddenGroups: new Set(['harness::turn']) })
const rows = [row('t-1', 'harness::turn')]
const dispatch = bar({ id: 'root', groupKey: 'harness::turn' })
const step = bar({ id: 'step', groupKey: 'harness::turn step' })
expect(
ids(reconcileTraceVisibility(verdicts, [dispatch, step], rows, sel)),
).toEqual(['t-1'])
// `step` pruned from the feed; only the hidden dispatch bar remains.
expect(
ids(reconcileTraceVisibility(verdicts, [dispatch], rows, sel)),
).toEqual(['t-1'])
})

it('keeps a live trace visible even while every bar in the feed is hidden', () => {
// A running turn: worker spans only reach the feed when they CLOSE, so
// early on the feed holds nothing but hidden dispatch plumbing. The
// row must not hide while the trace is live.
const verdicts = new Map<string, boolean>()
const sel = selection({ hiddenGroups: new Set(['harness::turn']) })
const rows = [row('t-1', 'harness::turn')]
const dispatch = bar({ id: 'root', groupKey: 'harness::turn' })
expect(
ids(
reconcileTraceVisibility(
verdicts,
[dispatch],
rows,
sel,
new Set(['t-1']),
),
),
).toEqual(['t-1'])
// Settled (no longer live) with a still-hidden composition → hides.
expect(
ids(reconcileTraceVisibility(verdicts, [dispatch], rows, sel)),
).toEqual([])
})

it('drops cached verdicts once the trace leaves the list too', () => {
const verdicts = new Map<string, boolean>()
const sel = selection({ hiddenGroups: new Set(['fn']) })
Expand Down Expand Up @@ -306,4 +351,49 @@ describe('mergeFetchedVerdicts', () => {
expect(verdicts.get('t-visible')).toBe(true)
expect(verdicts.has('t-hidden')).toBe(false)
})

it('never downgrades a visible verdict — the read raced a newer feed frame', () => {
const verdicts = new Map<string, boolean>([['t-1', true]])
mergeFetchedVerdicts(
verdicts,
['t-1'],
[
stored({
span_id: 'h1',
attributes: [['function_id', 'fn']],
}),
],
sel,
)
expect(verdicts.get('t-1')).toBe(true)
})
})

describe('liveTraceIds', () => {
// Realistic epoch ms — `toMs` sniffs nano vs ms by magnitude.
const now = 1_700_000_000_000

it('counts a pending snapshot as live', () => {
const live = liveTraceIds(
[stored({ span_id: 's1', trace_id: 't-1', pending: true })],
now,
)
expect(live.has('t-1')).toBe(true)
})

it('counts a just-ended span as live, an old one as settled', () => {
const justEnded = stored({
span_id: 's1',
trace_id: 't-recent',
end_time_unix_nano: (now - 2_000) * 1e6,
})
const longSettled = stored({
span_id: 's2',
trace_id: 't-old',
end_time_unix_nano: (now - 60_000) * 1e6,
})
const live = liveTraceIds([justEnded, longSettled], now)
expect(live.has('t-recent')).toBe(true)
expect(live.has('t-old')).toBe(false)
})
})
94 changes: 79 additions & 15 deletions console/web/src/pages/TracesV2/hooks/useSpanFilteredTraceRows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,26 @@
// read) never keep a row alive, so a kept row always opens to a non-empty
// detail.
//
// Feed verdicts recompute on every frame (a live trace must flip visible
// the moment a surviving span arrives — the first frames of a turn are all
// dispatch plumbing) and STICK once the trace prunes out of the feed: a
// settled trace's composition never changes. Fetched verdicts land in the
// same cache. The cache resets when the selection changes; a row stays
// visible while its verdict is unknown or its read failed — better to
// show a hideable row than to hide real work on a guess.
// Under a FIXED selection visibility is MONOTONE: spans only accumulate
// in the store, so a trace with one visible span has it forever. A `true`
// verdict therefore sticks; only unknown traces start at `false` and flip
// the moment a surviving span arrives (the first frames of a turn are all
// dispatch plumbing). Without the stick, a long run would vanish mid-way:
// the feed retains ~2min, so a quiet stretch (one long LLM/tool call)
// prunes the visible bars while hidden bookkeeping keeps the trace in the
// feed — and a feed-only recompute would flip the row back to hidden.
// Fetched verdicts land in the same cache. The cache resets when the
// selection changes; a row stays visible while its verdict is unknown or
// its read failed — better to show a hideable row than to hide real work
// on a guess.
//
// LIVE traces get one more guard: worker spans reach the store only when
// they CLOSE, so a running turn's composition is structurally incomplete —
// its first visible span may be an LLM call that takes minutes to close.
// A negative verdict is a guess there, so a row whose trace is still live
// (pending span in the feed, or a span end within the last few seconds)
// stays visible regardless. Bookkeeping traces settle in well under a
// second, so the exemption never resurfaces them.

import { useEffect, useMemo, useRef, useState } from 'react'
import { fetchTraces, type StoredSpan } from '../api/traces'
Expand All @@ -48,6 +61,7 @@ import {
spanFilterGroupKey,
storedSpansToTimelineSpans,
} from '../lib/timelineSpans'
import { isPendingSpan, toMs } from '../lib/traceTransform'
import { getWorkerName } from '../lib/traceUtils'
import type { TraceListItem } from './useTraceData'

Expand All @@ -57,6 +71,34 @@ import type { TraceListItem } from './useTraceData'
const FETCH_TRACE_CHUNK = 20
const FETCH_SPAN_LIMIT = 10_000

/** How recently a span of the trace must have ENDED for the trace to still
* count as live when no pending snapshot is in the feed (engines with
* `live_spans` off). Generous on purpose — liveness only defers hiding. */
const LIVE_END_SLACK_MS = 10_000

/**
* Traces with work still in flight, judged from the raw feed: a pending
* snapshot (the engine's queue wrapper stays pending for a turn's whole
* lifetime — see `followTurn.ts`), or a span that ended moments ago.
* Exported for tests.
*/
export function liveTraceIds(
spans: readonly StoredSpan[],
now: number,
): ReadonlySet<string> {
const live = new Set<string>()
for (const span of spans) {
if (live.has(span.trace_id)) continue
if (
isPendingSpan(span) ||
now - toMs(span.end_time_unix_nano) <= LIVE_END_SLACK_MS
) {
live.add(span.trace_id)
}
}
return live
}

/**
* The row's ROOT span rendered as filter keys, mirroring how
* `storedSpansToTimelineSpans` keys the same span in the feed (a listed
Expand All @@ -82,22 +124,29 @@ export function rowRootFilterKeys(row: TraceListItem): SpanFilterKeys {
* One reconcile pass: refresh `verdicts` from the bars currently in the
* feed, seed visible-root verdicts from the rows themselves, drop entries
* for traces gone from both the feed and the list, and return the rows
* that survive (unknown verdicts stay visible). Mutates `verdicts` — the
* hook owns the map across renders. Exported for tests.
* that survive (unknown verdicts stay visible, and so do rows in
* `liveTraces` — a running trace's negative verdict is a guess, its
* visible spans may simply not have closed yet). Verdicts are monotone
* within a selection: `true` sticks, `false` re-evaluates. Mutates
* `verdicts` — the hook owns the map across renders. Exported for tests.
*/
export function reconcileTraceVisibility(
verdicts: Map<string, boolean>,
bars: readonly TimelineSpan[],
rows: readonly TraceListItem[],
selection: SpanFilterSelection,
liveTraces: ReadonlySet<string> = new Set(),
): readonly TraceListItem[] {
const inFeed = new Set<string>()
for (const bar of bars) {
const traceId = bar.traceId
if (!traceId) continue
if (!inFeed.has(traceId)) {
inFeed.add(traceId)
verdicts.set(traceId, false)
// Monotone: a known-visible trace stays visible (spans only
// accumulate); only unknown traces start hidden pending a
// surviving bar.
if (!verdicts.has(traceId)) verdicts.set(traceId, false)
}
if (!verdicts.get(traceId) && !isSpanBarHidden(bar, selection)) {
verdicts.set(traceId, true)
Expand All @@ -118,7 +167,9 @@ export function reconcileTraceVisibility(
verdicts.delete(traceId)
}
}
const kept = rows.filter((r) => verdicts.get(r.traceId) !== false)
const kept = rows.filter(
(r) => verdicts.get(r.traceId) !== false || liveTraces.has(r.traceId),
)
// Identity-stable when nothing is hidden, so downstream memos hold.
return kept.length === rows.length ? rows : kept
}
Expand Down Expand Up @@ -159,7 +210,9 @@ function verdictBars(spans: readonly StoredSpan[]): TimelineSpan[] {
* longer has spans for counts as hidden (its detail would be empty). A
* truncated response (span count at the limit) only trusts POSITIVE
* verdicts: a visible span proves visibility, but "all hidden" might just
* mean the visible spans were cut off. Exported for tests.
* mean the visible spans were cut off. A `true` verdict already in the
* cache is never downgraded — the read raced a feed frame that proved
* visibility after the snapshot was taken. Exported for tests.
*/
export function mergeFetchedVerdicts(
verdicts: Map<string, boolean>,
Expand All @@ -177,7 +230,9 @@ export function mergeFetchedVerdicts(
const truncated = spans.length >= spanLimit
for (const traceId of requested) {
if (visible.has(traceId)) verdicts.set(traceId, true)
else if (!truncated) verdicts.set(traceId, false)
else if (!truncated && !verdicts.get(traceId)) {
verdicts.set(traceId, false)
}
}
}

Expand Down Expand Up @@ -207,15 +262,24 @@ export function useSpanFilteredTraceRows(
const [fetchTick, setFetchTick] = useState(0)

// fetchTick isn't read in the body — it signals `cache.verdicts` grew.
// The liveness set is derived from the feed at reconcile time (feed
// frames arrive continuously while anything is live, so it stays fresh
// without its own clock).
// biome-ignore lint/correctness/useExhaustiveDependencies: see above
const visibleRows = useMemo(() => {
let cache = cacheRef.current
if (!cache || cache.selection !== selection) {
cache = { selection, verdicts: new Map(), requested: new Set() }
cacheRef.current = cache
}
return reconcileTraceVisibility(cache.verdicts, bars, rows, selection)
}, [rows, bars, selection, fetchTick])
return reconcileTraceVisibility(
cache.verdicts,
bars,
rows,
selection,
liveTraceIds(feedSpans, Date.now()),
)
}, [rows, bars, feedSpans, selection, fetchTick])

// Composition reads for the rows neither source could judge: hidden
// root, no feed coverage. Runs after the memo above, so feed verdicts
Expand Down
Loading
Loading