diff --git a/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordSkeleton.tsx b/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordSkeleton.tsx new file mode 100644 index 0000000000..de90383bbc --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordSkeleton.tsx @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Flex, Stack } from '@nvidia/foundations-react-core'; +import { RecordSection } from '@studio/components/AnonymizerRecordView/RecordSection'; +import { StackedSkeleton } from '@studio/components/StackedSkeleton'; +import type { FC } from 'react'; + +const SKELETON_LINES = 8; + +const SkeletonBlock: FC = () => ( + + + +); + +interface AnonymizerRecordSkeletonProps { + readonly outputHeading: string; +} + +export const AnonymizerRecordSkeleton: FC = ({ outputHeading }) => ( + + + + + + + + + + + + + +); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordView.test.tsx b/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordView.test.tsx new file mode 100644 index 0000000000..e74eab4a1c --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordView.test.tsx @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AnonymizerRecordView } from '@studio/components/AnonymizerRecordView/AnonymizerRecordView'; +import { buildAnonymizerRecord } from '@studio/components/AnonymizerRecordView/parse'; +import { traceRow } from '@studio/components/AnonymizerRecordView/testFixtures'; +import { render, screen, waitFor } from '@testing-library/react'; +import type { FC } from 'react'; +import { MemoryRouter, useLocation } from 'react-router'; + +const SearchProbe: FC = () => {useLocation().search}; + +const renderRecord = ( + row: Record, + outputHeading = 'Replaced', + initialEntry = '/' +) => + render( + + + + + ); + +describe('AnonymizerRecordView', () => { + it('tags detected entities in both columns', () => { + renderRecord(traceRow); + + expect(screen.getAllByText('Original').length).toBeGreaterThan(0); + expect(screen.getByText('Replaced')).toBeInTheDocument(); + expect(screen.getAllByText('Bobby').length).toBeGreaterThan(0); + expect(screen.getAllByText('first_name').length).toBeGreaterThan(0); + expect(screen.getAllByText('45').length).toBeGreaterThan(0); + }); + + it('lists every replacement in the map', () => { + renderRecord(traceRow); + + expect(screen.getByText('Replacement Map')).toBeInTheDocument(); + expect(screen.getAllByText('Teddy').length).toBeGreaterThan(0); + expect(screen.getAllByText('age').length).toBeGreaterThan(0); + }); + + it('uses the caller heading for the output column', () => { + renderRecord(traceRow, 'Rewritten'); + + expect(screen.getByText('Rewritten')).toBeInTheDocument(); + }); + + it('still shows the map when the shared page param outruns a short record', () => { + renderRecord(traceRow, 'Replaced', '/?page=3'); + + expect(screen.getAllByText('Teddy').length).toBeGreaterThan(0); + expect(screen.queryByText('No Entries Found')).not.toBeInTheDocument(); + }); + + it('rewinds the shared page param instead of leaving it out of range', async () => { + renderRecord(traceRow, 'Replaced', '/?page=3'); + + await waitFor(() => expect(screen.getByTestId('search')).toHaveTextContent('')); + expect(screen.getByTestId('search').textContent).toBe(''); + }); + + it('explains when nothing was replaced', () => { + renderRecord({ biography: 'Nothing sensitive here.', biography_replaced: '' }); + + expect(screen.getByText('No entities were replaced in this record.')).toBeInTheDocument(); + expect(screen.getByText('No output was produced for this record.')).toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordView.tsx b/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordView.tsx new file mode 100644 index 0000000000..5c6ab4d1d9 --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/AnonymizerRecordView.tsx @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Flex, Stack, Text } from '@nvidia/foundations-react-core'; +import { HighlightedText } from '@studio/components/AnonymizerRecordView/HighlightedText'; +import { RecordSection } from '@studio/components/AnonymizerRecordView/RecordSection'; +import { ReplacementMapTable } from '@studio/components/AnonymizerRecordView/ReplacementMapTable'; +import type { AnonymizerRecord } from '@studio/components/AnonymizerRecordView/types'; +import { memo, type FC } from 'react'; + +interface AnonymizerRecordViewProps { + readonly record: AnonymizerRecord; + readonly outputHeading: string; +} + +export const AnonymizerRecordView: FC = memo( + ({ record, outputHeading }) => ( + + + + + + + + + + + + {record.replacements.length ? ( + + ) : ( + + No entities were replaced in this record. + + )} + + + ) +); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/HighlightedText.tsx b/web/packages/studio/src/components/AnonymizerRecordView/HighlightedText.tsx new file mode 100644 index 0000000000..d86fe10e04 --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/HighlightedText.tsx @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Badge, Text } from '@nvidia/foundations-react-core'; +import type { TextSegment } from '@studio/components/AnonymizerRecordView/types'; +import { entityTagColor } from '@studio/routes/AnonymizerBuilderRoute/constants'; +import { memo, type FC } from 'react'; + +interface HighlightedTextProps { + readonly segments: readonly TextSegment[]; + readonly emptyMessage: string; +} + +export const HighlightedText: FC = memo(({ segments, emptyMessage }) => + segments.length ? ( + + {segments.map((segment, index) => { + if (!segment.label) return {segment.text}; + const color = entityTagColor(segment.label); + return ( + + + {segment.text} + + + {segment.label} + + + ); + })} + + ) : ( + + {emptyMessage} + + ) +); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/RecordSection.tsx b/web/packages/studio/src/components/AnonymizerRecordView/RecordSection.tsx new file mode 100644 index 0000000000..b42fa371a8 --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/RecordSection.tsx @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Stack, Text } from '@nvidia/foundations-react-core'; +import type { FC, ReactNode } from 'react'; + +interface RecordSectionProps { + readonly heading: string; + readonly className?: string; + readonly children: ReactNode; +} + +/** Shared by the record view and its skeleton so the two stay aligned. */ +export const RecordSection: FC = ({ heading, className, children }) => ( + + + {heading} + + {children} + +); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/ReplacementMapTable.tsx b/web/packages/studio/src/components/AnonymizerRecordView/ReplacementMapTable.tsx new file mode 100644 index 0000000000..b0a6270c59 --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/ReplacementMapTable.tsx @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { Badge, Text } from '@nvidia/foundations-react-core'; +import type { EntityReplacement } from '@studio/components/AnonymizerRecordView/types'; +import { entityTagColor } from '@studio/routes/AnonymizerBuilderRoute/constants'; +import { ArrowRight } from 'lucide-react'; +import { memo, useCallback, useEffect, useMemo, type ComponentProps, type FC } from 'react'; + +const REPLACEMENTS_PAGE_SIZE = 20; +const ARROW_COLUMN_SIZE = 48; + +interface ReplacementMapTableProps { + readonly replacements: readonly EntityReplacement[]; +} + +export const ReplacementMapTable: FC = memo(({ replacements }) => { + const dataViewState = useStudioDataViewState({ defaultPageSize: REPLACEMENTS_PAGE_SIZE }); + + const { pageIndex: requestedPage, pageSize } = dataViewState.pagination.state; + // `page` is shared, so it outlives a record pager move to a map with fewer rows. + const lastPageIndex = Math.max(Math.ceil(replacements.length / pageSize) - 1, 0); + const pageIndex = Math.min(requestedPage, lastPageIndex); + + const setPagination = dataViewState.pagination.set; + useEffect(() => { + if (requestedPage > lastPageIndex) { + setPagination((prev) => ({ ...prev, pageIndex: lastPageIndex })); + } + }, [requestedPage, lastPageIndex, setPagination]); + + const pageRows = useMemo( + () => replacements.slice(pageIndex * pageSize, (pageIndex + 1) * pageSize), + [replacements, pageIndex, pageSize] + ); + + const makeColumns = useCallback< + ComponentProps>['makeColumns'] + >( + (col) => [ + col.accessor('label', { + header: 'Label', + cell: ({ getValue }) => ( + + {getValue()} + + ), + }), + col.accessor('original', { + header: 'Original', + cell: ({ getValue }) => {getValue()}, + }), + col.display({ + id: 'arrow', + header: '', + size: ARROW_COLUMN_SIZE, + cell: () => , + }), + col.accessor('synthetic', { + header: 'Replacement', + cell: ({ getValue }) => {getValue()}, + }), + ], + [] + ); + + return ( + + dataViewState={dataViewState} + makeColumns={makeColumns} + maxTwoLines={false} + attributes={{ + DataViewRoot: { data: pageRows, totalCount: replacements.length }, + DataViewPagination: { showWhileEmpty: false, showWhileLessThanPageSize: false }, + }} + /> + ); +}); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/parse.test.ts b/web/packages/studio/src/components/AnonymizerRecordView/parse.test.ts new file mode 100644 index 0000000000..efd2630561 --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/parse.test.ts @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildAnonymizerRecord, + buildReplacedEntities, + outputColumn, + parseEntities, + parseReplacements, + toSegments, + toUtf16Offsets, +} from '@studio/components/AnonymizerRecordView/parse'; +import { + entities, + ORIGINAL, + replacements, + REPLACED, + traceRow, +} from '@studio/components/AnonymizerRecordView/testFixtures'; + +describe('parseEntities', () => { + it('reads spans out of the wrapped entity list', () => { + expect(parseEntities(traceRow.final_entities)).toEqual(entities); + }); + + it('decodes cells that arrive as JSON strings', () => { + expect(parseEntities(JSON.stringify(traceRow.final_entities))).toEqual(entities); + }); + + it('drops entries missing positions', () => { + expect(parseEntities({ entities: [{ value: 'x', label: 'y' }] })).toEqual([]); + }); + + it('returns nothing for unusable cells', () => { + expect(parseEntities(undefined)).toEqual([]); + expect(parseEntities('not json')).toEqual([]); + expect(parseEntities({ entities: 'nope' })).toEqual([]); + }); +}); + +describe('parseReplacements', () => { + it('reads the replacement triples', () => { + expect(parseReplacements(traceRow._replacement_map)).toEqual(replacements); + }); + + it('drops entries that are not all strings', () => { + expect( + parseReplacements({ replacements: [{ original: 'a', label: 'b', synthetic: 7 }] }) + ).toEqual([]); + }); +}); + +describe('toSegments', () => { + it('splits text around the tagged spans', () => { + expect(toSegments(ORIGINAL, entities)).toEqual([ + { text: 'Bobby', label: 'first_name' }, + { text: ', a ' }, + { text: '40', label: 'age' }, + { text: '-year-old veterinarian.' }, + ]); + }); + + it('returns the whole text when nothing was detected', () => { + expect(toSegments(ORIGINAL, [])).toEqual([{ text: ORIGINAL }]); + }); + + it('skips spans that overlap an earlier one or run past the text', () => { + const overlapping = [ + { value: 'Bobby', label: 'first_name', start: 0, end: 5 }, + { value: 'obby', label: 'first_name', start: 1, end: 5 }, + { value: 'far', label: 'age', start: 900, end: 903 }, + ]; + expect(toSegments(ORIGINAL, overlapping)).toEqual([ + { text: 'Bobby', label: 'first_name' }, + { text: ', a 40-year-old veterinarian.' }, + ]); + }); +}); + +describe('buildReplacedEntities', () => { + it('positions each entity at its synthetic value in the replaced text', () => { + expect(buildReplacedEntities(entities, replacements, ORIGINAL, REPLACED)).toEqual([ + { value: 'Teddy', label: 'first_name', start: 0, end: 5 }, + { value: '45', label: 'age', start: 9, end: 11 }, + ]); + }); + + it('tracks offsets when a replacement changes length', () => { + const replaced = 'Bartholomew, a 45-year-old veterinarian.'; + const longer = [ + { original: 'Bobby', label: 'first_name', synthetic: 'Bartholomew' }, + ...replacements.slice(1), + ]; + expect(buildReplacedEntities(entities, longer, ORIGINAL, replaced)).toEqual([ + { value: 'Bartholomew', label: 'first_name', start: 0, end: 11 }, + { value: '45', label: 'age', start: 15, end: 17 }, + ]); + }); + + it('falls back to the original span when the synthetic value is absent', () => { + const unchanged = 'Bobby, a 45-year-old veterinarian.'; + expect(buildReplacedEntities(entities, replacements, ORIGINAL, unchanged)).toEqual([ + { value: 'Bobby', label: 'first_name', start: 0, end: 5 }, + { value: '45', label: 'age', start: 9, end: 11 }, + ]); + }); + + it('skips an identical literal that the replacement did not produce', () => { + const source = 'Teddy met Bobby'; + const output = 'Teddy met Teddy'; + const bobby = [{ value: 'Bobby', label: 'first_name', start: 10, end: 15 }]; + + expect(buildReplacedEntities(bobby, replacements, source, output)).toEqual([ + { value: 'Teddy', label: 'first_name', start: 10, end: 15 }, + ]); + }); + + it('skips an identical literal that a length-changing replacement moved past', () => { + const source = 'Alexander knows Teddy and Bobby'; + const output = 'Al knows Teddy and Teddy'; + const shifted = [ + { value: 'Alexander', label: 'first_name', start: 0, end: 9 }, + { value: 'Bobby', label: 'first_name', start: 26, end: 31 }, + ]; + const shorter = [ + { original: 'Alexander', label: 'first_name', synthetic: 'Al' }, + { original: 'Bobby', label: 'first_name', synthetic: 'Teddy' }, + ]; + + expect(buildReplacedEntities(shifted, shorter, source, output)).toEqual([ + { value: 'Al', label: 'first_name', start: 0, end: 2 }, + { value: 'Teddy', label: 'first_name', start: 19, end: 24 }, + ]); + }); + + it('matches case-insensitively when the map key differs in case', () => { + const mixedCase = [{ original: 'bobby', label: 'first_name', synthetic: 'Teddy' }]; + expect(buildReplacedEntities([entities[0]], mixedCase, ORIGINAL, REPLACED)).toEqual([ + { value: 'Teddy', label: 'first_name', start: 0, end: 5 }, + ]); + }); +}); + +describe('toUtf16Offsets', () => { + it('shifts offsets past a non-BMP character', () => { + const text = '😀 Alice'; + const detected = [{ value: 'Alice', label: 'first_name', start: 2, end: 7 }]; + + expect(toUtf16Offsets(text, detected)).toEqual([ + { value: 'Alice', label: 'first_name', start: 3, end: 8 }, + ]); + expect(toSegments(text, toUtf16Offsets(text, detected))).toEqual([ + { text: '😀 ' }, + { text: 'Alice', label: 'first_name' }, + ]); + }); + + it('leaves offsets alone when the text is all BMP', () => { + expect(toUtf16Offsets(ORIGINAL, entities)).toBe(entities); + }); + + it('drops spans that run past the last code point', () => { + const text = '😀 Alice'; + + expect( + toUtf16Offsets(text, [{ value: 'Alice', label: 'first_name', start: 2, end: 900 }]) + ).toEqual([]); + }); + + it('falls back to the replacement map when every converted span is dropped', () => { + const record = buildAnonymizerRecord( + { + biography: '😀 Bobby is a veterinarian.', + biography_replaced: '😀 Teddy is a veterinarian.', + final_entities: { + entities: [{ value: 'Bobby', label: 'first_name', start_position: 2, end_position: 900 }], + }, + _replacement_map: { replacements: [replacements[0]] }, + }, + 'biography' + ); + + expect(record.originalSegments).toEqual([ + { text: '😀 ' }, + { text: 'Bobby', label: 'first_name' }, + { text: ' is a veterinarian.' }, + ]); + }); +}); + +describe('outputColumn', () => { + it('finds the replace output', () => { + expect(outputColumn(traceRow, 'biography')).toBe('biography_replaced'); + }); + + it('prefers the rewrite output', () => { + expect(outputColumn({ text_rewritten: '', text_replaced: '' }, 'text')).toBe('text_rewritten'); + }); + + it('returns nothing when neither output is present', () => { + expect(outputColumn(traceRow, 'other')).toBeUndefined(); + }); +}); + +describe('buildAnonymizerRecord', () => { + it('builds both highlighted columns and the replacement map', () => { + const record = buildAnonymizerRecord(traceRow, 'biography'); + + expect(record.original).toBe(ORIGINAL); + expect(record.replaced).toBe(REPLACED); + expect(record.replacements).toEqual(replacements); + expect(record.originalSegments).toEqual([ + { text: 'Bobby', label: 'first_name' }, + { text: ', a ' }, + { text: '40', label: 'age' }, + { text: '-year-old veterinarian.' }, + ]); + expect(record.replacedSegments).toEqual([ + { text: 'Teddy', label: 'first_name' }, + { text: ', a ' }, + { text: '45', label: 'age' }, + { text: '-year-old veterinarian.' }, + ]); + }); + + it('derives spans from the replacement map when detection produced none', () => { + const record = buildAnonymizerRecord( + { biography: ORIGINAL, biography_replaced: REPLACED, _replacement_map: { replacements } }, + 'biography' + ); + + expect(record.originalSegments).toEqual([ + { text: 'Bobby', label: 'first_name' }, + { text: ', a ' }, + { text: '40', label: 'age' }, + { text: '-year-old veterinarian.' }, + ]); + expect(record.replacedSegments).toEqual([ + { text: 'Teddy', label: 'first_name' }, + { text: ', a ' }, + { text: '45', label: 'age' }, + { text: '-year-old veterinarian.' }, + ]); + }); + + it('reads the rewrite output column', () => { + const record = buildAnonymizerRecord( + { text: ORIGINAL, text_rewritten: 'A veterinarian in his forties.' }, + 'text' + ); + + expect(record.replaced).toBe('A veterinarian in his forties.'); + expect(record.replacements).toEqual([]); + }); + + it('is empty when the record has no text', () => { + const record = buildAnonymizerRecord({}, 'biography'); + + expect(record.originalSegments).toEqual([]); + expect(record.replacedSegments).toEqual([]); + }); +}); diff --git a/web/packages/studio/src/components/AnonymizerRecordView/parse.ts b/web/packages/studio/src/components/AnonymizerRecordView/parse.ts new file mode 100644 index 0000000000..74c48d1130 --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/parse.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + AnonymizerEntity, + AnonymizerRecord, + EntityReplacement, + TextSegment, +} from '@studio/components/AnonymizerRecordView/types'; +import { asRecord } from '@studio/util/guards'; + +const DETECTED_ENTITIES_COLUMN = '_detected_entities'; +const FINAL_ENTITIES_COLUMN = 'final_entities'; +const REPLACEMENT_MAP_COLUMN = '_replacement_map'; + +/** Rewrite writes `_rewritten`; the replace strategies write `_replaced`. */ +export const REWRITTEN_SUFFIX = '_rewritten'; +export const REPLACED_SUFFIX = '_replaced'; +export const OUTPUT_SUFFIXES = [REWRITTEN_SUFFIX, REPLACED_SUFFIX] as const; + +/** Trace cells arrive either already decoded or as a JSON string, depending on the writer. */ +const decodeCell = (value: unknown): unknown => { + if (typeof value !== 'string') return value; + try { + return JSON.parse(value) as unknown; + } catch { + return undefined; + } +}; + +const wrappedList = (cell: unknown, key: string): unknown[] => { + const entries = asRecord(decodeCell(cell))?.[key]; + return Array.isArray(entries) ? entries : []; +}; + +const toEntity = (entry: unknown): AnonymizerEntity | undefined => { + const row = asRecord(entry); + if (!row) return undefined; + const { value, label, start_position: start, end_position: end } = row; + if (typeof label !== 'string' || typeof start !== 'number' || typeof end !== 'number') { + return undefined; + } + return { value: typeof value === 'string' ? value : '', label, start, end }; +}; + +export const parseEntities = (cell: unknown): AnonymizerEntity[] => + wrappedList(cell, 'entities').flatMap((entry) => { + const entity = toEntity(entry); + return entity ? [entity] : []; + }); + +export const parseReplacements = (cell: unknown): EntityReplacement[] => + wrappedList(cell, 'replacements').flatMap((entry) => { + const row = asRecord(entry); + const { original, label, synthetic } = row ?? {}; + return typeof original === 'string' && + typeof label === 'string' && + typeof synthetic === 'string' + ? [{ original, label, synthetic }] + : []; + }); + +const byPosition = (a: AnonymizerEntity, b: AnonymizerEntity): number => + a.start - b.start || a.end - b.end; + +const SURROGATE_PAIR = /[\uD800-\uDBFF][\uDC00-\uDFFF]/; + +/** Detection counts code points the way Python does; string indices here are UTF-16 units. */ +export const toUtf16Offsets = ( + text: string, + entities: readonly AnonymizerEntity[] +): readonly AnonymizerEntity[] => { + if (!entities.length || !SURROGATE_PAIR.test(text)) return entities; + + const unitForCodePoint: number[] = []; + for (let index = 0; index < text.length; ) { + unitForCodePoint.push(index); + index += (text.codePointAt(index) ?? 0) > 0xffff ? 2 : 1; + } + unitForCodePoint.push(text.length); + + // Dropping rather than clamping: an offset past the last code point would map onto + // text.length and slip through the bounds check in toSegments. + return entities.flatMap((entity) => { + const start = unitForCodePoint[entity.start]; + const end = unitForCodePoint[entity.end]; + return start === undefined || end === undefined ? [] : [{ ...entity, start, end }]; + }); +}; + +export const toSegments = (text: string, entities: readonly AnonymizerEntity[]): TextSegment[] => { + if (!entities.length) return text ? [{ text }] : []; + + const segments: TextSegment[] = []; + let cursor = 0; + for (const entity of [...entities].sort(byPosition)) { + if (entity.start < cursor || entity.end <= entity.start || entity.end > text.length) continue; + if (entity.start > cursor) segments.push({ text: text.slice(cursor, entity.start) }); + segments.push({ text: text.slice(entity.start, entity.end), label: entity.label }); + cursor = entity.end; + } + if (cursor < text.length) segments.push({ text: text.slice(cursor) }); + return segments; +}; + +interface SyntheticLookups { + readonly byValueLabel: Map; + readonly byValue: Map; + readonly byValueLabelLower: Map; + readonly byValueLower: Map; +} + +// Label first so the newline separator stays unambiguous — labels never contain one, values may. +const lookupKey = (value: string, label: string): string => `${label}\n${value}`; + +const buildSyntheticLookups = (replacements: readonly EntityReplacement[]): SyntheticLookups => { + const lookups: SyntheticLookups = { + byValueLabel: new Map(), + byValue: new Map(), + byValueLabelLower: new Map(), + byValueLower: new Map(), + }; + for (const { original, label, synthetic } of replacements) { + const lower = original.toLowerCase(); + lookups.byValueLabel.set(lookupKey(original, label), synthetic); + lookups.byValue.set(original, synthetic); + lookups.byValueLabelLower.set(lookupKey(lower, label), synthetic); + lookups.byValueLower.set(lower, synthetic); + } + return lookups; +}; + +/** Exact value+label, then value only, then both again case-insensitively. */ +const resolveSynthetic = (entity: AnonymizerEntity, lookups: SyntheticLookups): string => { + const lower = entity.value.toLowerCase(); + return ( + lookups.byValueLabel.get(lookupKey(entity.value, entity.label)) ?? + lookups.byValue.get(entity.value) ?? + lookups.byValueLabelLower.get(lookupKey(lower, entity.label)) ?? + lookups.byValueLower.get(lower) ?? + entity.value + ); +}; + +/** Text between entities is untouched, so the gap since the last match predicts this one. */ +const locate = (text: string, needle: string, expected: number, searchFrom: number): number => { + if (!needle) return -1; + if (expected >= searchFrom && text.startsWith(needle, expected)) return expected; + return text.indexOf(needle, searchFrom); +}; + +/** Aligns each span through the unchanged gap, then falls back to a forward search. */ +export const buildReplacedEntities = ( + originalEntities: readonly AnonymizerEntity[], + replacements: readonly EntityReplacement[], + originalText: string, + replacedText: string +): AnonymizerEntity[] => { + const lookups = buildSyntheticLookups(replacements); + const replaced: AnonymizerEntity[] = []; + let originalCursor = 0; + let searchFrom = 0; + + for (const entity of [...originalEntities].sort(byPosition)) { + const { start, end, label } = entity; + if (start < originalCursor || end <= start || end > originalText.length) continue; + + const expected = searchFrom + (start - originalCursor); + const originalSpan = originalText.slice(start, end); + let synthetic = resolveSynthetic(entity, lookups); + let position = locate(replacedText, synthetic, expected, searchFrom); + if (position < 0 && synthetic !== originalSpan) { + position = locate(replacedText, originalSpan, expected, searchFrom); + if (position >= 0) synthetic = originalSpan; + } + + originalCursor = end; + if (position < 0) continue; + + replaced.push({ + value: replacedText.slice(position, position + synthetic.length), + label, + start: position, + end: position + synthetic.length, + }); + searchFrom = position + synthetic.length; + } + + return replaced; +}; + +const escapeRegExp = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** Fallback for when detection produced no spans but the replacement map did. */ +const entitiesByCaseInsensitiveSearch = ( + replacements: readonly EntityReplacement[], + text: string +): AnonymizerEntity[] => + replacements + .flatMap(({ original, label }) => { + if (!original || !label) return []; + return [...text.matchAll(new RegExp(escapeRegExp(original), 'gi'))].map((match) => ({ + value: match[0], + label, + start: match.index, + end: match.index + match[0].length, + })); + }) + .sort(byPosition); + +const entitiesBySyntheticSearch = ( + replacements: readonly EntityReplacement[], + text: string +): AnonymizerEntity[] => + replacements + .flatMap(({ synthetic, label }) => { + if (!synthetic || !label) return []; + return [...text.matchAll(new RegExp(escapeRegExp(synthetic), 'g'))].map((match) => ({ + value: synthetic, + label, + start: match.index, + end: match.index + synthetic.length, + })); + }) + .sort(byPosition); + +const asText = (value: unknown): string => (typeof value === 'string' ? value : ''); + +export const outputColumn = ( + row: Record, + textColumn: string +): string | undefined => + OUTPUT_SUFFIXES.map((suffix) => `${textColumn}${suffix}`).find((column) => column in row); + +export const buildAnonymizerRecord = ( + row: Record, + textColumn: string +): AnonymizerRecord => { + const original = asText(row[textColumn]); + const outputKey = outputColumn(row, textColumn); + const replaced = outputKey ? asText(row[outputKey]) : ''; + const replacements = parseReplacements(row[REPLACEMENT_MAP_COLUMN]); + + const detected = + FINAL_ENTITIES_COLUMN in row + ? parseEntities(row[FINAL_ENTITIES_COLUMN]) + : parseEntities(row[DETECTED_ENTITIES_COLUMN]); + const converted = toUtf16Offsets(original, detected); + const originalEntities = converted.length + ? converted + : entitiesByCaseInsensitiveSearch(replacements, original); + + const derived = buildReplacedEntities(originalEntities, replacements, original, replaced); + const replacedEntities = derived.length + ? derived + : entitiesBySyntheticSearch(replacements, replaced); + + return { + original, + replaced, + originalSegments: toSegments(original, originalEntities), + replacedSegments: toSegments(replaced, replacedEntities), + replacements, + }; +}; diff --git a/web/packages/studio/src/components/AnonymizerRecordView/testFixtures.ts b/web/packages/studio/src/components/AnonymizerRecordView/testFixtures.ts new file mode 100644 index 0000000000..0529eb48aa --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/testFixtures.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + AnonymizerEntity, + EntityReplacement, +} from '@studio/components/AnonymizerRecordView/types'; + +export const ORIGINAL = 'Bobby, a 40-year-old veterinarian.'; +export const REPLACED = 'Teddy, a 45-year-old veterinarian.'; + +/** The parsed shape. Kept literal so `parseEntities` is asserted against data, not a second mapping. */ +export const entities: AnonymizerEntity[] = [ + { value: 'Bobby', label: 'first_name', start: 0, end: 5 }, + { value: '40', label: 'age', start: 9, end: 11 }, +]; + +export const replacements: EntityReplacement[] = [ + { original: 'Bobby', label: 'first_name', synthetic: 'Teddy' }, + { original: '40', label: 'age', synthetic: '45' }, +]; + +export const traceRow = { + biography: ORIGINAL, + biography_replaced: REPLACED, + final_entities: { + entities: [ + { value: 'Bobby', label: 'first_name', start_position: 0, end_position: 5 }, + { value: '40', label: 'age', start_position: 9, end_position: 11 }, + ], + }, + _replacement_map: { replacements }, +}; diff --git a/web/packages/studio/src/components/AnonymizerRecordView/types.ts b/web/packages/studio/src/components/AnonymizerRecordView/types.ts new file mode 100644 index 0000000000..6ea52f826a --- /dev/null +++ b/web/packages/studio/src/components/AnonymizerRecordView/types.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface AnonymizerEntity { + readonly value: string; + readonly label: string; + readonly start: number; + readonly end: number; +} + +export interface EntityReplacement { + readonly original: string; + readonly label: string; + readonly synthetic: string; +} + +export interface TextSegment { + readonly text: string; + readonly label?: string; +} + +export interface AnonymizerRecord { + readonly original: string; + readonly replaced: string; + readonly originalSegments: readonly TextSegment[]; + readonly replacedSegments: readonly TextSegment[]; + readonly replacements: readonly EntityReplacement[]; +} diff --git a/web/packages/studio/src/components/NewDataDesignerJobForm/previewApi.ts b/web/packages/studio/src/components/NewDataDesignerJobForm/previewApi.ts index 64545bb328..ef6e758394 100644 --- a/web/packages/studio/src/components/NewDataDesignerJobForm/previewApi.ts +++ b/web/packages/studio/src/components/NewDataDesignerJobForm/previewApi.ts @@ -3,6 +3,7 @@ import type { DataDesignerConfig } from '@nemo/sdk/generated/data-designer/schema'; import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { readLineDelimitedStream } from '@studio/util/lineStream'; /** Request body for the data designer preview stream endpoint */ export interface PreviewRequestBody { @@ -81,21 +82,8 @@ export async function streamPreview( const body = response.body; if (!body) throw new Error('No response body'); - const reader = body.pipeThrough(new TextDecoderStream()).getReader(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += value; - const lines = buffer.split('\n'); - buffer = lines.pop() ?? ''; - for (const line of lines) { - const msg = parsePreviewLine(line.trim()); - if (msg) onLine(msg); - } - } - - const last = parsePreviewLine(buffer.trim()); - if (last) onLine(last); + await readLineDelimitedStream(body, (line) => { + const msg = parsePreviewLine(line.trim()); + if (msg) onLine(msg); + }); } diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/AnonymizerBuilderForm.tsx b/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/AnonymizerBuilderForm.tsx index 9b949bdc03..f670759ff2 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/AnonymizerBuilderForm.tsx +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/AnonymizerBuilderForm.tsx @@ -5,7 +5,7 @@ import { useAnonymizerCreateRunJob, useAnonymizerListEntityLabels, } from '@nemo/sdk/generated/anonymizer/api'; -import type { RunJob } from '@nemo/sdk/generated/anonymizer/schema'; +import type { PreviewRequest, RunJob } from '@nemo/sdk/generated/anonymizer/schema'; import { Banner, Button, @@ -14,7 +14,6 @@ import { Panel, SegmentedControl, Stack, - Text, } from '@nvidia/foundations-react-core'; import { getErrorMessage } from '@studio/api/common/utils'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; @@ -24,28 +23,37 @@ import { DataSourceSection } from '@studio/routes/AnonymizerBuilderRoute/compone import { EntitiesSection } from '@studio/routes/AnonymizerBuilderRoute/components/EntitiesSection'; import { GenerationSection } from '@studio/routes/AnonymizerBuilderRoute/components/GenerationSection'; import { ModelSettingsSection } from '@studio/routes/AnonymizerBuilderRoute/components/ModelSettingsSection'; +import { PreviewPanel } from '@studio/routes/AnonymizerBuilderRoute/components/PreviewPanel'; +import { + PANEL_TABS, + TAB_MODEL_SETTINGS, + TAB_SOURCE, +} from '@studio/routes/AnonymizerBuilderRoute/constants'; import { buildAnonymizerJobRequest, + buildAnonymizerPreviewRequest, type AnonymizerFormData, } from '@studio/routes/AnonymizerBuilderRoute/schema'; +import { useAnonymizerPreview } from '@studio/routes/AnonymizerBuilderRoute/useAnonymizerPreview'; import { useDefaultRoleModels } from '@studio/routes/AnonymizerBuilderRoute/useDefaultRoleModels'; +import { + outputHeadingForStrategy, + tabForValidationErrors, +} from '@studio/routes/AnonymizerBuilderRoute/utils'; import { getWorkspaceAnonymizerRoute, getWorkspaceJobDetailRoute } from '@studio/routes/utils'; -import { useState, type FC } from 'react'; -import { useFormContext } from 'react-hook-form'; +import { useCallback, useState, type FC } from 'react'; +import { useFormContext, useWatch, type FieldErrors } from 'react-hook-form'; +import { useAuth } from 'react-oidc-context'; import { useNavigate } from 'react-router'; -const TAB_SOURCE = 'source'; -const TAB_MODEL_SETTINGS = 'model-settings'; - -const PANEL_TABS = [ - { value: TAB_SOURCE, children: 'Source' }, - { value: TAB_MODEL_SETTINGS, children: 'Model Settings' }, -]; +const INCOMPLETE_FORM_MESSAGE = 'Please complete the required fields highlighted below.'; export const AnonymizerBuilderForm: FC = () => { const navigate = useNavigate(); const workspace = useWorkspaceFromPath(); + const { user } = useAuth(); const form = useFormContext(); + const strategy = useWatch({ control: form.control, name: 'strategy' }); const [activeTab, setActiveTab] = useState(TAB_SOURCE); const [submitError, setSubmitError] = useState(undefined); @@ -78,22 +86,50 @@ export const AnonymizerBuilderForm: FC = () => { }, }); - const onSubmit = form.handleSubmit( - (values) => { - setSubmitError(undefined); - createJob.mutate({ - workspace, - data: buildAnonymizerJobRequest(values, defaultEntityLabels?.data ?? []), - }); - }, - (errors) => { - const onlyModelErrors = Object.keys(errors).every((key) => key === 'roleModels'); - setActiveTab(onlyModelErrors ? TAB_MODEL_SETTINGS : TAB_SOURCE); - setSubmitError('Please complete the required fields highlighted below.'); - } + const showValidationErrors = useCallback((errors: FieldErrors) => { + setActiveTab(tabForValidationErrors(Object.keys(errors))); + setSubmitError(INCOMPLETE_FORM_MESSAGE); + }, []); + + // Not `trigger`: `formState` is a proxy tracking what the render body reads, so its `errors` + // are empty here. Only `handleSubmit` hands back a populated set. + const getPreviewRequest = useCallback( + () => + new Promise((resolve) => { + form + .handleSubmit( + (values) => { + setSubmitError(undefined); + resolve(buildAnonymizerPreviewRequest(values, defaultEntityLabels?.data ?? [])); + }, + (errors) => { + showValidationErrors(errors); + resolve(undefined); + } + )() + .catch(() => { + setSubmitError(INCOMPLETE_FORM_MESSAGE); + resolve(undefined); + }); + }), + [form, defaultEntityLabels, showValidationErrors] ); - const handleCancel = () => navigate(getWorkspaceAnonymizerRoute(workspace)); + const preview = useAnonymizerPreview({ + workspace, + accessToken: user?.access_token ?? undefined, + getRequest: getPreviewRequest, + }); + + const onSubmit = form.handleSubmit((values) => { + setSubmitError(undefined); + createJob.mutate({ + workspace, + data: buildAnonymizerJobRequest(values, defaultEntityLabels?.data ?? []), + }); + }, showValidationErrors); + + const isBusy = createJob.isPending || isLoadingModels || isLoadingEntityLabels; return (
@@ -103,40 +139,37 @@ export const AnonymizerBuilderForm: FC = () => { elevation="high" density="standard" attributes={{ PanelContent: { className: 'flex-1 min-h-0 overflow-auto' } }} - slotFooter={ - - - - - } > - + + + {preview.isPreviewing ? ( + + ) : ( + + )} + - {submitError && ( + {submitError ? ( {submitError} - )} + ) : null}
@@ -155,9 +188,15 @@ export const AnonymizerBuilderForm: FC = () => { - - Your records preview will appear here - + + Full Run + + } + /> ); diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/GenerationSection.tsx b/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/GenerationSection.tsx index 6894c9b4c9..c1289bd619 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/GenerationSection.tsx +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/GenerationSection.tsx @@ -6,6 +6,7 @@ import { ControlledTextInput } from '@nemo/common/src/components/form/Controlled import { Stack, Text } from '@nvidia/foundations-react-core'; import { StrategyParamsSection } from '@studio/routes/AnonymizerBuilderRoute/components/StrategyParamsSection'; import { + MAX_PREVIEW_ROWS, STRATEGY_DESCRIPTIONS, STRATEGY_OPTIONS, } from '@studio/routes/AnonymizerBuilderRoute/constants'; @@ -31,6 +32,7 @@ export const GenerationSection: FC = () => { = ({ + preview, + pendingOutputHeading, + slotActions, +}) => { + const { result, logs, isPreviewing, error, hasRun, wasStopped } = preview; + const { records, textColumn, failedRecords } = result; + const [recordIndex, setRecordIndex] = useState(0); + const [pagedRecords, setPagedRecords] = useState(records); + + // Reset during render, not in an effect, so a new result never paints the old index first. + if (pagedRecords !== records) { + setPagedRecords(records); + setRecordIndex(0); + } + + const activeRow = records[recordIndex]; + const { record, outputHeading } = useMemo( + () => ({ + record: activeRow ? buildAnonymizerRecord(activeRow, textColumn) : undefined, + outputHeading: outputColumn(activeRow ?? {}, textColumn)?.endsWith(REWRITTEN_SUFFIX) + ? OUTPUT_HEADING_REWRITTEN + : OUTPUT_HEADING_REPLACED, + }), + [activeRow, textColumn] + ); + + // Rebuilt per streamed log frame otherwise, collapsed accordion or not. + const logItems = useMemo( + () => [ + { + value: 'logs', + slotTrigger: 'Logs', + slotContent: ( + + ), + }, + ], + [logs] + ); + + return ( + + Preview + {records.length > 0 ? ( + + ) : null} + {slotActions} + + } + slotFooter={logs.length > 0 ? : null} + > + + {error ? ( + + {error} + + ) : null} + {failedRecords.length > 0 ? ( + + {failedRecords.length} record(s) failed during the preview run. + + ) : null} + {record ? ( + + ) : isPreviewing ? ( + + ) : error ? null : ( + + + {wasStopped + ? 'Preview stopped before any records arrived.' + : hasRun + ? 'The preview run returned no records.' + : 'Your records preview will appear here'} + + + )} + + + ); +}; diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/RecordPager.tsx b/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/RecordPager.tsx new file mode 100644 index 0000000000..71f6676910 --- /dev/null +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/components/RecordPager.tsx @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Button, Flex, Text } from '@nvidia/foundations-react-core'; +import { ChevronLeft, ChevronRight } from 'lucide-react'; +import type { FC } from 'react'; + +const ICON_SIZE = 16; + +interface RecordPagerProps { + readonly index: number; + readonly total: number; + readonly onChange: (index: number) => void; +} + +export const RecordPager: FC = ({ index, total, onChange }) => ( + + + + Record {index + 1} of {total} + + + +); diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/constants.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/constants.ts index dec66eefe4..3135bb0602 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/constants.ts +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/constants.ts @@ -232,6 +232,8 @@ export const entityTagColor = (label: string): EntityTagColor => COLOR_BY_LABEL.get(label) ?? ENTITY_CUSTOM_TAG_COLOR; export const DEFAULT_PREVIEW_ROWS = 1; +/** Mirrors the plugin's `preview_num_records.max`, which 422s rather than clamping. */ +export const MAX_PREVIEW_ROWS = 10; export const MAX_COLUMN_INTROSPECTION_BYTES = 50 * 1024 * 1024; @@ -279,3 +281,13 @@ export const activeRolesForStrategy = (strategy: Strategy): string[] => { if (strategy === STRATEGY_SUBSTITUTE) return [...DETECTION_ROLES, REPLACE_ROLE]; return [...DETECTION_ROLES]; }; + +export const TAB_SOURCE = 'source'; +export const TAB_MODEL_SETTINGS = 'model-settings'; + +export type BuilderTab = typeof TAB_SOURCE | typeof TAB_MODEL_SETTINGS; + +export const PANEL_TABS: { value: BuilderTab; children: string }[] = [ + { value: TAB_SOURCE, children: 'Source' }, + { value: TAB_MODEL_SETTINGS, children: 'Model Settings' }, +]; diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/previewApi.test.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/previewApi.test.ts new file mode 100644 index 0000000000..a3e190213f --- /dev/null +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/previewApi.test.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { parsePreviewFrame } from '@studio/routes/AnonymizerBuilderRoute/previewApi'; + +describe('parsePreviewFrame', () => { + it('reads log frames', () => { + expect(parsePreviewFrame('{"kind":"log","level":"warning","message":"slow"}')).toEqual({ + kind: 'log', + level: 'warning', + message: 'slow', + }); + }); + + it('defaults an unknown log level to info', () => { + expect(parsePreviewFrame('{"kind":"log","level":"trace","message":"x"}')).toEqual({ + kind: 'log', + level: 'info', + message: 'x', + }); + }); + + it('reads the trace dataset and its text column', () => { + const line = '{"kind":"trace_dataset","records":[{"a":1}],"original_text_column":"biography"}'; + + expect(parsePreviewFrame(line)).toEqual({ + kind: 'trace_dataset', + records: [{ a: 1 }], + original_text_column: 'biography', + }); + }); + + it('leaves the text column undefined when the server omits it', () => { + expect(parsePreviewFrame('{"kind":"trace_dataset","records":[]}')).toEqual({ + kind: 'trace_dataset', + records: [], + original_text_column: undefined, + }); + }); + + it('drops record entries that are not objects', () => { + expect(parsePreviewFrame('{"kind":"failed_records","records":[1,{"b":2}]}')).toEqual({ + kind: 'failed_records', + records: [{ b: 2 }], + }); + }); + + it('reads the control frames', () => { + expect(parsePreviewFrame('{"kind":"done"}')).toEqual({ kind: 'done' }); + expect(parsePreviewFrame('{"kind":"heartbeat"}')).toEqual({ kind: 'heartbeat' }); + expect(parsePreviewFrame('{"kind":"error","message":"boom"}')).toEqual({ + kind: 'error', + message: 'boom', + }); + }); + + it('ignores blank lines, malformed JSON, and unknown frame kinds', () => { + expect(parsePreviewFrame(' ')).toBeUndefined(); + expect(parsePreviewFrame('{not json')).toBeUndefined(); + expect(parsePreviewFrame('[1,2]')).toBeUndefined(); + expect(parsePreviewFrame('{"kind":"something_new"}')).toBeUndefined(); + expect(parsePreviewFrame('{"records":[]}')).toBeUndefined(); + }); +}); diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/previewApi.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/previewApi.ts new file mode 100644 index 0000000000..0e21b6d69c --- /dev/null +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/previewApi.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + LogFrameLevel, + type Done, + type Error as ErrorFrame, + type FailedRecordsFrame, + type Heartbeat, + type LogFrame, + type PreviewDatasetFrame, + type PreviewRequest, + type TraceDatasetFrame, +} from '@nemo/sdk/generated/anonymizer/schema'; +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { asRecord } from '@studio/util/guards'; +import { readLineDelimitedStream } from '@studio/util/lineStream'; + +export type PreviewFrame = + | LogFrame + | PreviewDatasetFrame + | TraceDatasetFrame + | FailedRecordsFrame + | Heartbeat + | Done + | ErrorFrame; + +const LOG_LEVELS: readonly string[] = Object.values(LogFrameLevel); + +const asRecordList = (value: unknown): Record[] => + Array.isArray(value) + ? value.flatMap((entry) => { + const row = asRecord(entry); + return row ? [row] : []; + }) + : []; + +/** Frames arrive as NDJSON. Anything unrecognised is dropped rather than surfaced as an error. */ +export const parsePreviewFrame = (line: string): PreviewFrame | undefined => { + const trimmed = line.trim(); + if (!trimmed) return undefined; + + let decoded: unknown; + try { + decoded = JSON.parse(trimmed); + } catch { + return undefined; + } + + const frame = asRecord(decoded); + const kind = frame?.kind; + if (!frame || typeof kind !== 'string') return undefined; + + switch (kind) { + case 'log': { + const { level, message } = frame; + return { + kind, + level: LOG_LEVELS.includes(String(level)) + ? (level as LogFrame['level']) + : LogFrameLevel.info, + message: typeof message === 'string' ? message : '', + }; + } + case 'preview_dataset': + case 'failed_records': + return { kind, records: asRecordList(frame.records) }; + case 'trace_dataset': { + const column = frame.original_text_column; + return { + kind, + records: asRecordList(frame.records), + original_text_column: typeof column === 'string' ? column : undefined, + }; + } + case 'heartbeat': + case 'done': + return { kind }; + case 'error': + return { + kind, + message: typeof frame.message === 'string' ? frame.message : 'The preview run failed.', + }; + default: + return undefined; + } +}; + +/** FastAPI returns `detail` as either a plain string or a list of pydantic errors. */ +const messageFromErrorBody = (body: string): string | undefined => { + let decoded: unknown; + try { + decoded = JSON.parse(body); + } catch { + return body.trim() || undefined; + } + const detail = asRecord(decoded)?.detail; + if (typeof detail === 'string') return detail; + if (!Array.isArray(detail)) return undefined; + const messages = detail.flatMap((item) => { + const msg = asRecord(item)?.msg; + return typeof msg === 'string' ? [msg] : []; + }); + return messages.length ? messages.join(' ') : undefined; +}; + +const previewPath = (workspace: string): string => + `/apis/anonymizer/v2/workspaces/${encodeURIComponent(workspace)}/preview`; + +export const streamAnonymizerPreview = async ( + workspace: string, + request: PreviewRequest, + accessToken: string | undefined, + signal: AbortSignal, + onFrame: (frame: PreviewFrame) => void +): Promise => { + const response = await fetch(`${PLATFORM_BASE_URL}${previewPath(workspace)}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + 'X-Source': 'NeMo Studio', + }, + body: JSON.stringify(request), + signal, + }); + + if (!response.ok) { + const body = await response.text(); + throw new Error(messageFromErrorBody(body) ?? `Preview failed: ${response.status}`); + } + if (!response.body) throw new Error('The preview response was empty.'); + + await readLineDelimitedStream(response.body, (line) => { + const frame = parsePreviewFrame(line); + if (frame) onFrame(frame); + }); +}; diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.test.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.test.ts index 47e7e40b6c..959955222a 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.test.ts +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.test.ts @@ -3,6 +3,7 @@ import { DETECTION_ROLES, + MAX_PREVIEW_ROWS, REPLACE_ROLE, REWRITE_ROLES, ROLE_LABELS, @@ -279,6 +280,13 @@ describe('anonymizerFormSchema', () => { expect(result.error?.issues.some((i) => i.path.join('.') === 'entityLabels')).toBe(true); }); + it('rejects more preview rows than the server accepts', () => { + const result = parse({ previewRows: MAX_PREVIEW_ROWS + 1 }); + expect(result.success).toBe(false); + expect(result.error?.issues.some((i) => i.path.join('.') === 'previewRows')).toBe(true); + expect(parse({ previewRows: MAX_PREVIEW_ROWS }).success).toBe(true); + }); + it('accepts custom mode with labels, or with defaults included', () => { expect( parse({ entityMode: 'custom', includeDefaultEntities: false, entityLabels: ['email'] }) diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.ts index 7f4fdfbf22..2fc5b52374 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.ts +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/schema.ts @@ -5,6 +5,7 @@ import { generateDefaultName } from '@nemo/common/src/utils/generateDefaultName' import type { AnonymizerConfigInput, ModelConfig, + PreviewRequest, Rewrite, RunJobRequest, SelectedModelsOverrides, @@ -16,6 +17,7 @@ import { DEFAULT_MODEL_MAX_TOKENS, DEFAULT_MODEL_TIMEOUT_SECONDS, DEFAULT_PREVIEW_ROWS, + MAX_PREVIEW_ROWS, ENTITY_MODE_CUSTOM, HASH_ALGORITHM_DEFAULT, HASH_ALGORITHM_VALUES, @@ -53,7 +55,7 @@ export const anonymizerFormSchema = z sourceType: z.enum(['url', 'dataset']), source: z.string().trim().min(1, 'A data source is required'), strategy: z.enum(['substitute', 'redact', 'annotate', 'hash', 'rewrite']), - previewRows: z.number().int().min(1), + previewRows: z.number().int().min(1).max(MAX_PREVIEW_ROWS), textColumn: z.string().optional(), dataSummary: z.string().optional(), entityMode: z.enum([ENTITY_MODE_CUSTOM, 'auto']), @@ -272,3 +274,11 @@ export const buildAnonymizerJobRequest = ( }, }; }; + +export const buildAnonymizerPreviewRequest = ( + form: AnonymizerFormData, + defaultEntityLabels: string[] = [] +): PreviewRequest => ({ + ...buildAnonymizerJobRequest(form, defaultEntityLabels).spec, + num_records: form.previewRows, +}); diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/useAnonymizerPreview.test.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/useAnonymizerPreview.test.ts new file mode 100644 index 0000000000..332c8ace0a --- /dev/null +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/useAnonymizerPreview.test.ts @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PreviewRequest } from '@nemo/sdk/generated/anonymizer/schema'; +import { streamAnonymizerPreview } from '@studio/routes/AnonymizerBuilderRoute/previewApi'; +import { useAnonymizerPreview } from '@studio/routes/AnonymizerBuilderRoute/useAnonymizerPreview'; +import { act, renderHook, waitFor } from '@testing-library/react'; + +vi.mock('@studio/routes/AnonymizerBuilderRoute/previewApi', () => ({ + streamAnonymizerPreview: vi.fn(), +})); + +const streamMock = vi.mocked(streamAnonymizerPreview); + +const requestFor = (name: string): PreviewRequest => ({ name }) as unknown as PreviewRequest; + +const deferred = (): { promise: Promise; resolve: (value: T) => void } => { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +}; + +const renderPreview = (getRequest: () => Promise) => + renderHook(() => useAnonymizerPreview({ workspace: 'default', accessToken: 't', getRequest })); + +beforeEach(() => { + streamMock.mockReset(); +}); + +describe('useAnonymizerPreview', () => { + it('ignores a run whose request resolves after a newer one started', async () => { + const first = deferred(); + const second = deferred(); + const pending = [first.promise, second.promise]; + const getRequest = vi.fn(() => pending.shift() ?? Promise.resolve(undefined)); + + streamMock.mockImplementation(async (_workspace, request, _token, _signal, onFrame) => { + onFrame({ + kind: 'trace_dataset', + records: [{ from: (request as unknown as { name: string }).name }], + original_text_column: 'text', + }); + }); + + const { result } = renderPreview(getRequest); + + act(() => { + void result.current.runPreview(); + void result.current.runPreview(); + }); + + await act(async () => { + second.resolve(requestFor('second')); + first.resolve(requestFor('first')); + await Promise.resolve(); + }); + + await waitFor(() => expect(streamMock).toHaveBeenCalledTimes(1)); + expect(result.current.result.records).toEqual([{ from: 'second' }]); + }); + + it('surfaces an error when building the request throws', async () => { + const getRequest = vi.fn(() => Promise.reject(new Error('Could not validate the form.'))); + + const { result } = renderPreview(getRequest); + + await act(async () => { + await result.current.runPreview(); + }); + + expect(result.current.error).toBe('Could not validate the form.'); + expect(streamMock).not.toHaveBeenCalled(); + }); + + it('stays idle when the form has no request to preview', async () => { + const getRequest = vi.fn(() => Promise.resolve(undefined)); + + const { result } = renderPreview(getRequest); + + await act(async () => { + await result.current.runPreview(); + }); + + expect(result.current.hasRun).toBe(false); + expect(result.current.isPreviewing).toBe(false); + expect(streamMock).not.toHaveBeenCalled(); + }); +}); diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/useAnonymizerPreview.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/useAnonymizerPreview.ts new file mode 100644 index 0000000000..635bfd5e4b --- /dev/null +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/useAnonymizerPreview.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isAbortError } from '@nemo/common/src/components/AssistantChat/completionUtils'; +import type { PreviewRequest } from '@nemo/sdk/generated/anonymizer/schema'; +import { + streamAnonymizerPreview, + type PreviewFrame, +} from '@studio/routes/AnonymizerBuilderRoute/previewApi'; +import { startTransition, useCallback, useEffect, useRef, useState } from 'react'; + +const DEFAULT_TEXT_COLUMN = 'text'; + +interface PreviewResult { + readonly records: Record[]; + readonly textColumn: string; + readonly failedRecords: Record[]; +} + +const EMPTY_RESULT: PreviewResult = { + records: [], + textColumn: DEFAULT_TEXT_COLUMN, + failedRecords: [], +}; + +export interface UseAnonymizerPreviewOptions { + readonly workspace: string; + readonly accessToken: string | undefined; + /** Returns the request to preview, or undefined when the form isn't ready. */ + readonly getRequest: () => Promise; +} + +export interface UseAnonymizerPreview { + readonly result: PreviewResult; + readonly logs: readonly string[]; + readonly isPreviewing: boolean; + readonly error: string | undefined; + readonly hasRun: boolean; + readonly wasStopped: boolean; + readonly runPreview: () => Promise; + /** The server unwinds on disconnect, but an in-flight model call still finishes. */ + readonly stopPreview: () => void; +} + +export const useAnonymizerPreview = ({ + workspace, + accessToken, + getRequest, +}: UseAnonymizerPreviewOptions): UseAnonymizerPreview => { + const [result, setResult] = useState(EMPTY_RESULT); + const [logs, setLogs] = useState([]); + const [isPreviewing, setIsPreviewing] = useState(false); + const [error, setError] = useState(undefined); + const [hasRun, setHasRun] = useState(false); + const [wasStopped, setWasStopped] = useState(false); + const abortRef = useRef(null); + const runIdRef = useRef(0); + + useEffect(() => () => abortRef.current?.abort(), []); + + const stopPreview = useCallback(() => { + if (!abortRef.current) return; + setWasStopped(true); + abortRef.current.abort(); + }, []); + + const runPreview = useCallback(async () => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + const runId = ++runIdRef.current; + const isCurrent = (): boolean => runIdRef.current === runId; + + const onFrame = (frame: PreviewFrame) => { + if (!isCurrent()) return; + switch (frame.kind) { + case 'log': + // One frame per read is too far apart for React to batch; keeps Stop responsive. + startTransition(() => { + if (!isCurrent()) return; + setLogs((prev) => [...prev, frame.message]); + }); + break; + case 'trace_dataset': + setResult((prev) => ({ + ...prev, + records: frame.records, + textColumn: frame.original_text_column ?? prev.textColumn, + })); + break; + case 'failed_records': + setResult((prev) => ({ ...prev, failedRecords: frame.records })); + break; + case 'error': + setError(frame.message); + break; + default: + break; + } + }; + + try { + const request = await getRequest(); + if (!isCurrent()) return; + if (!request) { + abortRef.current = null; + return; + } + + setResult(EMPTY_RESULT); + setLogs([]); + setError(undefined); + setHasRun(true); + setWasStopped(false); + setIsPreviewing(true); + + await streamAnonymizerPreview(workspace, request, accessToken, controller.signal, onFrame); + } catch (err) { + if (isAbortError(err) || !isCurrent()) return; + setError((err instanceof Error && err.message) || 'The preview run failed.'); + } finally { + if (abortRef.current === controller) { + abortRef.current = null; + setIsPreviewing(false); + } + } + }, [workspace, accessToken, getRequest]); + + return { result, logs, isPreviewing, error, hasRun, wasStopped, runPreview, stopPreview }; +}; diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.test.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.test.ts index e515307e8f..a9e90acf1d 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.test.ts +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.test.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import type { DataDesignerModelOption } from '@studio/components/NewDataDesignerJobForm/utils'; -import { isGlinerModel } from '@studio/routes/AnonymizerBuilderRoute/utils'; +import { + isGlinerModel, + outputHeadingForStrategy, + tabForValidationErrors, +} from '@studio/routes/AnonymizerBuilderRoute/utils'; const model = (name: string, servedModelName: string, id = name): DataDesignerModelOption => ({ id, name, served_model_name: servedModelName }) as DataDesignerModelOption; @@ -27,3 +31,32 @@ describe('isGlinerModel', () => { expect(isGlinerModel(model('gpt-oss-120b', 'openai/gpt-oss-120b'))).toBe(false); }); }); + +describe('tabForValidationErrors', () => { + it('stays on Source whenever a Source field failed', () => { + expect(tabForValidationErrors(['source'])).toBe('source'); + expect(tabForValidationErrors(['source', 'roleModels'])).toBe('source'); + expect(tabForValidationErrors(['entityLabels', 'roleModels'])).toBe('source'); + }); + + it('switches to Model Settings only when models are the sole failure', () => { + expect(tabForValidationErrors(['roleModels'])).toBe('model-settings'); + }); + + it('stays on Source when no fields are reported', () => { + expect(tabForValidationErrors([])).toBe('source'); + }); +}); + +describe('outputHeadingForStrategy', () => { + it('names the rewrite output', () => { + expect(outputHeadingForStrategy('rewrite')).toBe('Rewritten'); + }); + + it('names the replace output for every other strategy', () => { + expect(outputHeadingForStrategy('substitute')).toBe('Replaced'); + expect(outputHeadingForStrategy('redact')).toBe('Replaced'); + expect(outputHeadingForStrategy('annotate')).toBe('Replaced'); + expect(outputHeadingForStrategy('hash')).toBe('Replaced'); + }); +}); diff --git a/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.ts b/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.ts index 583d675a53..3017d816e1 100644 --- a/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.ts +++ b/web/packages/studio/src/routes/AnonymizerBuilderRoute/utils.ts @@ -2,6 +2,26 @@ // SPDX-License-Identifier: Apache-2.0 import type { DataDesignerModelOption } from '@studio/components/NewDataDesignerJobForm/utils'; +import { + STRATEGY_REWRITE, + TAB_MODEL_SETTINGS, + TAB_SOURCE, + type BuilderTab, + type Strategy, +} from '@studio/routes/AnonymizerBuilderRoute/constants'; + +export const OUTPUT_HEADING_REPLACED = 'Replaced'; +export const OUTPUT_HEADING_REWRITTEN = 'Rewritten'; export const isGlinerModel = (model: DataDesignerModelOption): boolean => /gliner/i.test(model.name) || /gliner/i.test(model.served_model_name ?? ''); + +/** Only `roleModels` lives on Model Settings; an empty list must not read as "models only". */ +export const tabForValidationErrors = (fields: readonly string[]): BuilderTab => + fields.length > 0 && fields.every((field) => field === 'roleModels') + ? TAB_MODEL_SETTINGS + : TAB_SOURCE; + +/** The output column only exists once results land, so the skeleton reads the strategy instead. */ +export const outputHeadingForStrategy = (strategy: Strategy): string => + strategy === STRATEGY_REWRITE ? OUTPUT_HEADING_REWRITTEN : OUTPUT_HEADING_REPLACED; diff --git a/web/packages/studio/src/routes/AnonymizerJobDetailRoute/util.ts b/web/packages/studio/src/routes/AnonymizerJobDetailRoute/util.ts index 154b3ad8e0..8b29bfd83b 100644 --- a/web/packages/studio/src/routes/AnonymizerJobDetailRoute/util.ts +++ b/web/packages/studio/src/routes/AnonymizerJobDetailRoute/util.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { RunJob } from '@nemo/sdk/generated/anonymizer/schema'; +import { OUTPUT_SUFFIXES } from '@studio/components/AnonymizerRecordView/parse'; export const ANONYMIZER_POLLING_INTERVAL_MS = 5000; @@ -39,9 +40,6 @@ export const metadataTextColumn = (metadata: string | undefined): string | undef } }; -/** Rewrite writes `_rewritten`; the replace strategies write `_replaced`. */ -const OUTPUT_SUFFIXES = ['_rewritten', '_replaced']; - export const orderResultColumns = (columns: string[], textColumn: string | undefined): string[] => { if (!textColumn || !columns.includes(textColumn)) return columns; const output = OUTPUT_SUFFIXES.map((suffix) => `${textColumn}${suffix}`).find((column) => diff --git a/web/packages/studio/src/util/guards.ts b/web/packages/studio/src/util/guards.ts new file mode 100644 index 0000000000..4c9a56f0e6 --- /dev/null +++ b/web/packages/studio/src/util/guards.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Narrows parsed JSON to a keyed object, excluding arrays and null. */ +export const asRecord = (value: unknown): Record | undefined => + typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; diff --git a/web/packages/studio/src/util/lineStream.test.ts b/web/packages/studio/src/util/lineStream.test.ts new file mode 100644 index 0000000000..5ac2d91159 --- /dev/null +++ b/web/packages/studio/src/util/lineStream.test.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readLineDelimitedStream } from '@studio/util/lineStream'; + +const streamOf = (chunks: string[]): NonNullable => { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(controller) { + chunks.forEach((chunk) => controller.enqueue(encoder.encode(chunk))); + controller.close(); + }, + }); +}; + +const collect = async (chunks: string[]): Promise => { + const lines: string[] = []; + await readLineDelimitedStream(streamOf(chunks), (line) => lines.push(line)); + return lines; +}; + +describe('readLineDelimitedStream', () => { + it('splits a single chunk on newlines', async () => { + expect(await collect(['a\nb\nc'])).toEqual(['a', 'b', 'c']); + }); + + it('rejoins a line split across chunk boundaries', async () => { + expect(await collect(['{"ki', 'nd":"do', 'ne"}\n'])).toEqual(['{"kind":"done"}', '']); + }); + + it('holds back the trailing partial until the next chunk completes it', async () => { + expect(await collect(['a\nb', 'c\nd'])).toEqual(['a', 'bc', 'd']); + }); + + it('flushes whatever is left at end of stream', async () => { + expect(await collect(['only'])).toEqual(['only']); + }); + + it('emits an empty final line for a newline-terminated body', async () => { + expect(await collect(['a\n'])).toEqual(['a', '']); + }); + + it('handles an empty body', async () => { + expect(await collect([])).toEqual(['']); + }); +}); diff --git a/web/packages/studio/src/util/lineStream.ts b/web/packages/studio/src/util/lineStream.ts new file mode 100644 index 0000000000..53c8fe0c43 --- /dev/null +++ b/web/packages/studio/src/util/lineStream.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read a newline-delimited response body, invoking `onLine` per line. A read rarely lands on a + * line boundary, so the trailing partial is held back until the next chunk completes it, then + * flushed at EOF. Callers decide what a blank line means. + */ +export const readLineDelimitedStream = async ( + body: NonNullable, + onLine: (line: string) => void +): Promise => { + const reader = body.pipeThrough(new TextDecoderStream()).getReader(); + let buffer = ''; + + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += value; + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + lines.forEach(onLine); + } + + onLine(buffer); +};