From 923e1d7f4df8e5a5401b6fe4b4b35990981dfee8 Mon Sep 17 00:00:00 2001 From: Sherry-hue <37186915+Sherry-hue@users.noreply.github.com> Date: Thu, 28 May 2026 17:18:14 +0800 Subject: [PATCH] feat(a2ui): add loading component instead of loading text --- .github/a2ui-catalog.instructions.md | 3 + .../a2ui-playground/lynx-src/a2ui/App.tsx | 2 + .../genui/a2ui-playground/src/catalog/a2ui.ts | 35 +++ .../src/hooks/useConversation.ts | 13 +- packages/genui/a2ui/etc/genui-a2ui.api.md | 7 + packages/genui/a2ui/package.json | 5 + .../genui/a2ui/src/catalog/Loading/index.tsx | 29 ++ packages/genui/a2ui/src/catalog/index.ts | 1 + packages/genui/a2ui/src/index.ts | 1 + .../genui/a2ui/src/react/A2UIRenderer.tsx | 21 +- packages/genui/a2ui/styles/catalog/Button.css | 10 +- .../genui/a2ui/styles/catalog/Loading.css | 61 ++++ packages/genui/server/agent/a2ui-catalog.ts | 3 + .../genui/server/agent/a2ui-stream-parser.ts | 155 ++++++++-- packages/genui/server/agent/a2ui-validator.ts | 26 -- .../server/agent/catalog/Loading/catalog.json | 14 + packages/genui/server/agent/image-resolver.ts | 271 ++++++++++++++---- packages/genui/server/app/a2ui/_shared.ts | 80 ++++++ .../genui/server/app/a2ui/action/route.ts | 15 +- .../server/app/a2ui/action/stream/route.ts | 51 +++- .../genui/server/app/a2ui/stream/route.ts | 31 +- 21 files changed, 685 insertions(+), 149 deletions(-) create mode 100644 packages/genui/a2ui/src/catalog/Loading/index.tsx create mode 100644 packages/genui/a2ui/styles/catalog/Loading.css create mode 100644 packages/genui/server/agent/catalog/Loading/catalog.json diff --git a/.github/a2ui-catalog.instructions.md b/.github/a2ui-catalog.instructions.md index 373ef6dd3f..abe1edf9e0 100644 --- a/.github/a2ui-catalog.instructions.md +++ b/.github/a2ui-catalog.instructions.md @@ -33,6 +33,9 @@ When a GenUI package builds a CLI or other generated artifact that another works When implementing A2UI v0.9 functions in `packages/genui/a2ui`, keep function resolution scoped to the active catalog first, with the global `FunctionRegistry` only as an escape hatch. Dynamic component props, checks, and function-call actions should all go through the same `resolveDynamicValue` / `executeFunctionCall` path so data bindings, nested function calls, zod argument coercion from `@a2ui/web_core`, and `formatString` data-context interpolation stay consistent. When verifying `packages/genui/a2ui-playground`, remember that `pnpm -F @lynx-js/genui-a2ui build` first runs `tsc --project tsconfig.build.json` and then regenerates catalog JSON through `build:catalog`. The playground consumes `@lynx-js/genui/a2ui` through package exports under `dist/**`, so you normally do not need a separate `tsc` step unless you intentionally skipped the package `build` step. +When streaming A2UI server responses, emit a root `Loading` component immediately after `createSurface` so the new surface has visible content before the model streams real components. Do not send `Image` components with unresolved search-query `url` strings directly to the renderer. Emit a same-id `Loading` component while `packages/genui/server/agent/image-resolver.ts` resolves the query to a loadable image URL, then emit an `updateComponents` message with the resolved `Image` so parent component references stay stable. + +When verifying `packages/genui/a2ui-playground`, remember that `pnpm -F @lynx-js/a2ui-reactlynx build` regenerates catalog JSON only. The playground consumes `@lynx-js/a2ui-reactlynx` through package exports under `dist/**`, so run `pnpm -F @lynx-js/a2ui-reactlynx exec tsc -p tsconfig.build.json` before rebuilding the playground if runtime TypeScript changed. For known A2UI playground examples, keep the web preview URL on `?demo=` instead of swapping it to the payload-store `messagesUrl`. `render.html` intentionally fetches known demo JSON in the browser shell and passes resolved messages into Lynx, avoiding fetch differences in the Lynx worker runtime; use payload-store URLs for custom edited JSON. diff --git a/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx b/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx index ace9044958..c26779d746 100644 --- a/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx +++ b/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx @@ -14,6 +14,7 @@ import { Image, LineChart, List, + Loading, Modal, PieChart, RadioGroup, @@ -76,6 +77,7 @@ const ALL_BUILTINS: readonly CatalogInput[] = [ manifestEntry(DateTimeInput, catalogManifests.DateTimeInput), manifestEntry(LineChart, catalogManifests.LineChart), manifestEntry(PieChart, catalogManifests.PieChart), + manifestEntry(Loading, catalogManifests.Loading), manifestEntry(RadioGroup, catalogManifests.RadioGroup), manifestEntry(Slider, catalogManifests.Slider), manifestEntry(TextField, catalogManifests.TextField), diff --git a/packages/genui/a2ui-playground/src/catalog/a2ui.ts b/packages/genui/a2ui-playground/src/catalog/a2ui.ts index c6c4ab2d9d..aac689b2fc 100644 --- a/packages/genui/a2ui-playground/src/catalog/a2ui.ts +++ b/packages/genui/a2ui-playground/src/catalog/a2ui.ts @@ -162,6 +162,41 @@ export const COMPONENT_CATALOG: ComponentDoc[] = [ openui: [], }, }, + { + name: 'Loading', + category: 'Display', + description: 'Shows an animated skeleton placeholder for pending content.', + props: schemaToProps(catalogManifests.Loading), + usage: { + a2ui: { + id: 'loading-state', + component: 'Loading', + variant: 'block', + }, + openui: {}, + }, + usageExamples: { + a2ui: [ + { + label: 'Inline', + value: { + id: 'inline-loading', + component: 'Loading', + variant: 'inline', + }, + }, + { + label: 'Block', + value: { + id: 'block-loading', + component: 'Loading', + variant: 'block', + }, + }, + ], + openui: [], + }, + }, { name: 'Icon', category: 'Display', diff --git a/packages/genui/a2ui-playground/src/hooks/useConversation.ts b/packages/genui/a2ui-playground/src/hooks/useConversation.ts index 5674e5f108..1f83ffcdf5 100644 --- a/packages/genui/a2ui-playground/src/hooks/useConversation.ts +++ b/packages/genui/a2ui-playground/src/hooks/useConversation.ts @@ -80,6 +80,15 @@ function cloneDataModel( } } +function cloneDataValue(value: unknown): unknown { + if (value === null || typeof value !== 'object') return value; + try { + return JSON.parse(JSON.stringify(value)) as unknown; + } catch { + return value; + } +} + function truncateConversationHistory( history: ModelChatMessage[], ): ModelChatMessage[] { @@ -108,7 +117,7 @@ function applyDataModel( delete model[key]; } if (value && typeof value === 'object' && !Array.isArray(value)) { - Object.assign(model, value as Record); + Object.assign(model, cloneDataValue(value) as Record); } return; } @@ -129,7 +138,7 @@ function applyDataModel( if (value === undefined) { delete cursor[last]; } else { - cursor[last] = value; + cursor[last] = cloneDataValue(value); } } diff --git a/packages/genui/a2ui/etc/genui-a2ui.api.md b/packages/genui/a2ui/etc/genui-a2ui.api.md index e8abb1792e..035d04ca10 100644 --- a/packages/genui/a2ui/etc/genui-a2ui.api.md +++ b/packages/genui/a2ui/etc/genui-a2ui.api.md @@ -324,6 +324,12 @@ export function LineChart(props: LineChartProps): ReactNode; // @public (undocumented) export function List(props: ListProps): ReactNode; +// Warning: (ae-forgotten-export) The symbol "LoadingProps" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "Loading" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function Loading(props: LoadingProps): ReactNode; + // Warning: (ae-missing-release-tag) "mergeCatalogs" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -611,6 +617,7 @@ export function useResolvedProps(properties: Record, surface: S // dist/catalog/Image/index.d.ts:4:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration // dist/catalog/LineChart/index.d.ts:9:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration // dist/catalog/List/index.d.ts:4:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration +// dist/catalog/Loading/index.d.ts:4:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration // dist/catalog/Modal/index.d.ts:4:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration // dist/catalog/PieChart/index.d.ts:9:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration // dist/catalog/RadioGroup/index.d.ts:4:4 - (tsdoc-undefined-tag) The TSDoc tag "@a2uiCatalog" is not defined in this configuration diff --git a/packages/genui/a2ui/package.json b/packages/genui/a2ui/package.json index 1bfc336184..6843ae30de 100644 --- a/packages/genui/a2ui/package.json +++ b/packages/genui/a2ui/package.json @@ -55,6 +55,11 @@ "default": "./dist/catalog/List/index.js" }, "./catalog/List/catalog.json": "./dist/catalog/List/catalog.json", + "./catalog/Loading": { + "types": "./dist/catalog/Loading/index.d.ts", + "default": "./dist/catalog/Loading/index.js" + }, + "./catalog/Loading/catalog.json": "./dist/catalog/Loading/catalog.json", "./catalog/Card": { "types": "./dist/catalog/Card/index.d.ts", "default": "./dist/catalog/Card/index.js" diff --git a/packages/genui/a2ui/src/catalog/Loading/index.tsx b/packages/genui/a2ui/src/catalog/Loading/index.tsx new file mode 100644 index 0000000000..e2fdf74d0b --- /dev/null +++ b/packages/genui/a2ui/src/catalog/Loading/index.tsx @@ -0,0 +1,29 @@ +// Copyright 2026 The Lynx Authors. All rights reserved. +// Licensed under the Apache License Version 2.0 that can be found in the +// LICENSE file in the root directory of this source tree. +import type { GenericComponentProps } from '../../store/types.js'; + +import '../../../styles/catalog/Loading.css'; + +/** + * @a2uiCatalog Loading + */ +export interface LoadingProps extends GenericComponentProps { + /** Visual density for the skeleton placeholder. */ + variant?: 'inline' | 'block'; +} + +export function Loading( + props: LoadingProps, +): import('@lynx-js/react').ReactNode { + const variant = props.variant ?? 'inline'; + + return ( + + + {variant === 'block' + ? + : null} + + ); +} diff --git a/packages/genui/a2ui/src/catalog/index.ts b/packages/genui/a2ui/src/catalog/index.ts index 3d1173fd9f..91140adeda 100755 --- a/packages/genui/a2ui/src/catalog/index.ts +++ b/packages/genui/a2ui/src/catalog/index.ts @@ -40,6 +40,7 @@ export { Divider } from './Divider/index.jsx'; export { Icon } from './Icon/index.jsx'; export { Image } from './Image/index.jsx'; export { List } from './List/index.jsx'; +export { Loading } from './Loading/index.jsx'; export { Modal } from './Modal/index.jsx'; export { RadioGroup } from './RadioGroup/index.jsx'; export { Row } from './Row/index.jsx'; diff --git a/packages/genui/a2ui/src/index.ts b/packages/genui/a2ui/src/index.ts index c1b574965b..246c4cd5a6 100644 --- a/packages/genui/a2ui/src/index.ts +++ b/packages/genui/a2ui/src/index.ts @@ -97,6 +97,7 @@ export { LineChart, PieChart, List, + Loading, Modal, RadioGroup, Row, diff --git a/packages/genui/a2ui/src/react/A2UIRenderer.tsx b/packages/genui/a2ui/src/react/A2UIRenderer.tsx index b729298d69..191d6b761d 100644 --- a/packages/genui/a2ui/src/react/A2UIRenderer.tsx +++ b/packages/genui/a2ui/src/react/A2UIRenderer.tsx @@ -8,6 +8,8 @@ import { useA2UIContext } from './useA2UIContext.js'; import { useAction } from './useAction.js'; import { useCatalog } from './useCatalog.js'; import { splitUnsupportedProps, useResolvedProps } from './useDataBinding.js'; +import { Loading } from '../catalog/Loading/index.jsx'; +import type { LoadingProps } from '../catalog/Loading/index.jsx'; import type { ComponentInstance, Resource, Surface } from '../store/types.js'; const noop = () => { @@ -29,23 +31,8 @@ export interface UnsupportedInfo { fields?: string[]; } -function DefaultLoading(props: { id: string }) { - const content = `loading ${props.id}...`; - return ( - - {content} - - ); +function DefaultLoading(_props: { id: string }) { + return Loading({ variant: 'block' } as LoadingProps); } function DefaultUnsupportedNotice(props: UnsupportedInfo) { diff --git a/packages/genui/a2ui/styles/catalog/Button.css b/packages/genui/a2ui/styles/catalog/Button.css index d8d4ff7bd9..068589a649 100644 --- a/packages/genui/a2ui/styles/catalog/Button.css +++ b/packages/genui/a2ui/styles/catalog/Button.css @@ -28,12 +28,12 @@ } .button-primary { - background-color: var(--a2ui-color-primary); - border-color: var(--a2ui-color-primary); - color: var(--a2ui-color-on-primary); + background-color: var(--a2ui-color-surface-strong); + border-color: var(--a2ui-color-border-strong); + color: var(--a2ui-color-on-surface); box-shadow: - 0 10px 24px rgba(0, 85, 217, 0.16), - 0 1px 0 rgba(255, 255, 255, 0.22) inset; + 0 10px 24px var(--a2ui-color-overlay), + 0 1px 0 rgba(255, 255, 255, 0.4) inset; } .button-borderless { diff --git a/packages/genui/a2ui/styles/catalog/Loading.css b/packages/genui/a2ui/styles/catalog/Loading.css new file mode 100644 index 0000000000..e3c1ce30c6 --- /dev/null +++ b/packages/genui/a2ui/styles/catalog/Loading.css @@ -0,0 +1,61 @@ +@import "../theme.css"; + +.loading { + --a2ui-loading-base: var(--a2ui-color-surface-muted); + --a2ui-loading-glow: rgba(255, 255, 255, 0.34); + --a2ui-loading-highlight: var(--a2ui-color-border-strong); + + display: flex; + flex-direction: column; + align-items: stretch; + justify-content: center; + flex-shrink: 0; + gap: 10px; +} + +.loading-inline { + width: 100%; + min-height: 14px; +} + +.loading-block { + width: 100%; + min-height: 48px; + padding: 6px 0; +} + +.loading-skeleton { + height: 14px; + border-radius: 999px; + background: linear-gradient( + 90deg, + var(--a2ui-loading-base) 0%, + var(--a2ui-loading-glow) 30%, + var(--a2ui-loading-highlight) 50%, + var(--a2ui-loading-glow) 70%, + var(--a2ui-loading-base) 100% + ); + background-size: 220% 100%; + animation-name: a2ui-loading-shimmer; + animation-duration: 1200ms; + animation-timing-function: ease-in-out; + animation-iteration-count: infinite; +} + +.loading-skeleton-primary { + width: 96%; +} + +.loading-skeleton-secondary { + width: 68%; +} + +@keyframes a2ui-loading-shimmer { + 0% { + background-position: 120% 0; + } + + 100% { + background-position: -120% 0; + } +} diff --git a/packages/genui/server/agent/a2ui-catalog.ts b/packages/genui/server/agent/a2ui-catalog.ts index 430eb5f60c..91995f4b02 100644 --- a/packages/genui/server/agent/a2ui-catalog.ts +++ b/packages/genui/server/agent/a2ui-catalog.ts @@ -15,6 +15,7 @@ import iconManifest from './catalog/Icon/catalog.json'; import imageManifest from './catalog/Image/catalog.json'; import lineChartManifest from './catalog/LineChart/catalog.json'; import listManifest from './catalog/List/catalog.json'; +import loadingManifest from './catalog/Loading/catalog.json'; import modalManifest from './catalog/Modal/catalog.json'; import radioGroupManifest from './catalog/RadioGroup/catalog.json'; import rowManifest from './catalog/Row/catalog.json'; @@ -89,6 +90,7 @@ const CATALOG_MANIFESTS = [ rowManifest, columnManifest, listManifest, + loadingManifest, cardManifest, tabsManifest, modalManifest, @@ -117,6 +119,7 @@ const COMPONENT_SUMMARIES: Record = { Image: 'Display an image by URL.', LineChart: 'Display one or more numeric line series over shared labels.', List: 'Repeating layout container, commonly bound to a data path.', + Loading: 'Animated progress indicator for pending content.', Modal: 'Modal dialog with a trigger component and a content component. The trigger opens the modal locally when tapped.', RadioGroup: 'Single-choice selector for a list of string options.', diff --git a/packages/genui/server/agent/a2ui-stream-parser.ts b/packages/genui/server/agent/a2ui-stream-parser.ts index 626e11a83e..e3cdc0be91 100644 --- a/packages/genui/server/agent/a2ui-stream-parser.ts +++ b/packages/genui/server/agent/a2ui-stream-parser.ts @@ -3,6 +3,7 @@ // LICENSE file in the root directory of this source tree. import type { A2UIMessage } from './a2ui-validator'; +import { isLoadableImageSource } from './image-resolver'; type A2UIUpdateComponentsMessage = Extract< A2UIMessage, @@ -85,22 +86,22 @@ function isA2UIComponent(value: unknown): value is Record & { && value.component.length > 0; } -function isLoadableImageSource(value: unknown): value is string { - if (typeof value !== 'string') return false; - const src = value.trim(); - if (!src) return false; - if (/^(?:https?:|data:image\/|blob:|file:)/iu.test(src)) return true; - if (/^(?:\/|\.\/|\.\.\/)/u.test(src)) return true; - return /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/iu.test(src); -} - -function isStreamRenderableComponent( - component: Record, -): boolean { - if (component.component !== 'Image') return true; +function toStreamRenderableComponent( + component: Record & { id: string; component: string }, + pendingImagePaths?: Set, +): Record & { id: string; component: string } { + if (component.component !== 'Image') return component; const url = component.url; - if (isRecord(url) && typeof url.path === 'string') return true; - return isLoadableImageSource(url); + if (isRecord(url) && typeof url.path === 'string') { + if (!pendingImagePaths?.has(normalizePointer(url.path))) { + return component; + } + } else if (isLoadableImageSource(url)) return component; + return { + id: component.id, + component: 'Loading', + variant: 'block', + }; } function sniffUpdateComponentsSurfaceId(buffer: string): string | null { @@ -122,12 +123,89 @@ function placeholderId(id: string): string { return `loading_${id}`; } +function normalizePointer(path: string): string { + if (!path || path === '/') return '/'; + return path.startsWith('/') ? path : `/${path}`; +} + +function appendPointer(path: string, segment: string): string { + const encoded = segment.replace(/~/gu, '~0').replace(/\//gu, '~1'); + return path === '/' ? `/${encoded}` : `${path}/${encoded}`; +} + +function decodePointerSegment(segment: string): string { + return segment.replace(/~1/gu, '/').replace(/~0/gu, '~'); +} + +function lastPointerSegment(path: string): string { + const parts = normalizePointer(path).split('/').filter(Boolean); + const last = parts[parts.length - 1]; + return last ? decodePointerSegment(last) : ''; +} + +function isImageLikeKey(key: string): boolean { + return /(?:^|[-_])(?:image|photo|picture|avatar|cover|poster|artwork|thumbnail)(?:$|[-_])/iu + .test(key); +} + +function updatePendingImagePathsFromData( + value: unknown, + path: string, + pendingImagePaths: Set, +): void { + const normalizedPath = normalizePointer(path); + const key = lastPointerSegment(normalizedPath); + if (typeof value === 'string' && isImageLikeKey(key)) { + if (isLoadableImageSource(value)) { + pendingImagePaths.delete(normalizedPath); + } else { + pendingImagePaths.add(normalizedPath); + } + return; + } + + if (Array.isArray(value)) { + value.forEach((item, index) => + updatePendingImagePathsFromData( + item, + appendPointer(normalizedPath, String(index)), + pendingImagePaths, + ) + ); + return; + } + + if (!isRecord(value)) return; + for (const [childKey, child] of Object.entries(value)) { + updatePendingImagePathsFromData( + child, + appendPointer(normalizedPath, childKey), + pendingImagePaths, + ); + } +} + function createPlaceholderComponent(id: string): ComponentRecord { return { id: placeholderId(id), - component: 'Text', - text: 'Loading...', - variant: 'caption', + component: 'Loading', + variant: 'block', + }; +} + +function createSurfaceLoadingMessage(surfaceId: string): A2UIMessage { + return { + version: 'v0.9', + updateComponents: { + surfaceId, + components: [ + { + id: ROOT_COMPONENT_ID, + component: 'Loading', + variant: 'block', + }, + ], + }, }; } @@ -271,6 +349,7 @@ export class A2UIProtocolMessageStreamParser { string, Map >(); + private pendingImagePathsBySurface = new Map>(); private createdSurfaceIds = new Set(); public push(chunk: string): A2UIMessage[] { @@ -329,8 +408,14 @@ export class A2UIProtocolMessageStreamParser { if (isA2UIMessage(parsed)) { if ('createSurface' in parsed && parsed.createSurface) { this.createdSurfaceIds.add(parsed.createSurface.surfaceId); - } - if (!isUpdateComponentsMessage(parsed)) { + messages.push(parsed); + messages.push( + createSurfaceLoadingMessage(parsed.createSurface.surfaceId), + ); + } else if ('updateDataModel' in parsed && parsed.updateDataModel) { + this.updatePendingImagePaths(parsed); + messages.push(parsed); + } else if (!isUpdateComponentsMessage(parsed)) { messages.push(parsed); } } @@ -375,15 +460,18 @@ export class A2UIProtocolMessageStreamParser { return; } if (!isA2UIComponent(parsed)) return; - if (!isStreamRenderableComponent(parsed)) return; const surfaceId = sniffUpdateComponentsSurfaceId( this.buffer.slice(0, start), ); if (!surfaceId) return; + const renderable = toStreamRenderableComponent( + parsed, + this.pendingImagePathsBySurface.get(surfaceId), + ); const seen = this.seenComponentsBySurface.get(surfaceId) ?? new Map(); - seen.set(parsed.id, parsed as ComponentRecord); + seen.set(renderable.id, renderable as ComponentRecord); this.seenComponentsBySurface.set(surfaceId, seen); const components = buildReachableComponentSnapshot(seen, { @@ -412,6 +500,29 @@ export class A2UIProtocolMessageStreamParser { }, }); } + + private updatePendingImagePaths( + message: Extract< + A2UIMessage, + { updateDataModel: unknown } + >, + ): void { + const dataModel = message.updateDataModel as { + surfaceId: string; + path?: string; + value?: unknown; + }; + if (!('value' in dataModel)) return; + const pendingImagePaths = this.pendingImagePathsBySurface.get( + dataModel.surfaceId, + ) ?? new Set(); + updatePendingImagePathsFromData( + dataModel.value, + dataModel.path ?? '/', + pendingImagePaths, + ); + this.pendingImagePathsBySurface.set(dataModel.surfaceId, pendingImagePaths); + } } export function splitA2UIProtocolMessages( diff --git a/packages/genui/server/agent/a2ui-validator.ts b/packages/genui/server/agent/a2ui-validator.ts index 991853dd50..f0cea2253e 100644 --- a/packages/genui/server/agent/a2ui-validator.ts +++ b/packages/genui/server/agent/a2ui-validator.ts @@ -94,21 +94,6 @@ function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } -function hasDispatchableAction(value: unknown): boolean { - if (!isRecord(value)) return false; - - const candidate = value as { event?: unknown; functionCall?: unknown }; - if (isRecord(candidate.functionCall)) { - const functionCall = candidate.functionCall as { call?: unknown }; - return typeof functionCall.call === 'string' - && functionCall.call.length > 0; - } - - if (!isRecord(candidate.event)) return false; - const event = candidate.event as { name?: unknown }; - return typeof event.name === 'string' && event.name.length > 0; -} - export interface ValidationResult { ok: boolean; messages: A2UIMessage[]; @@ -311,9 +296,6 @@ export function validateA2UIOutput( }); const knownComponents = new Set(catalog.components.map((c) => c.name)); const componentSpecs = new Map(catalog.components.map((c) => [c.name, c])); - const requiresAction = new Set( - catalog.components.filter((c) => c.requiresAction).map((c) => c.name), - ); // structural checks ---------------------------------------------------- const firstMessage = messages[0]; @@ -381,14 +363,6 @@ export function validateA2UIOutput( ); } bucket.set(comp.id, comp); - if (requiresAction.has(comp.component)) { - const action = comp.action; - if (!hasDispatchableAction(action)) { - errors.push( - `${comp.component} (id=${comp.id}) MUST carry action.event.name or action.functionCall.call.`, - ); - } - } const componentPaths: string[] = []; collectPaths(comp, componentPaths); for (const path of componentPaths) { diff --git a/packages/genui/server/agent/catalog/Loading/catalog.json b/packages/genui/server/agent/catalog/Loading/catalog.json new file mode 100644 index 0000000000..00cb356d9c --- /dev/null +++ b/packages/genui/server/agent/catalog/Loading/catalog.json @@ -0,0 +1,14 @@ +{ + "Loading": { + "properties": { + "variant": { + "type": "string", + "enum": [ + "inline", + "block" + ] + } + }, + "required": [] + } +} diff --git a/packages/genui/server/agent/image-resolver.ts b/packages/genui/server/agent/image-resolver.ts index bf98ac0ed6..0a29f7a28a 100644 --- a/packages/genui/server/agent/image-resolver.ts +++ b/packages/genui/server/agent/image-resolver.ts @@ -10,6 +10,25 @@ interface ImagePathRef { fallbackQuery: string; } +interface ImageDataPatch extends Record { + surfaceId: string; + path: string; + value: string; +} + +type A2UIUpdateComponentsMessage = Extract< + A2UIMessage, + { updateComponents: unknown } +>; +type A2UIComponent = A2UIUpdateComponentsMessage['updateComponents'][ + 'components' +][number]; + +interface PendingImageLoadingResult { + messages: A2UIMessage[]; + replacementCount: number; +} + interface PexelsPhoto { src?: { large2x?: string; @@ -63,28 +82,81 @@ const imageCache = new LruCache>( IMAGE_CACHE_MAX_ENTRIES, ); +export function isLoadableImageSource(value: unknown): value is string { + if (typeof value !== 'string') return false; + const src = value.trim(); + if (!src) return false; + if (/^(?:https?:|data:image\/|blob:|file:)/iu.test(src)) return true; + if (/^(?:\/|\.\/|\.\.\/)/u.test(src)) return true; + return /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/iu.test(src); +} + +export function replacePendingA2UIImagesWithLoading( + messages: A2UIMessage[], +): PendingImageLoadingResult { + const cloned = cloneMessages(messages); + let replacementCount = 0; + + for (const message of cloned) { + if (!('updateComponents' in message) || !message.updateComponents) { + continue; + } + const { components } = message.updateComponents; + for (let i = 0; i < components.length; i++) { + const component = components[i]; + if (component.component !== 'Image') continue; + const record = component as Record; + if (isLoadableImageSource(record.url)) continue; + if (isRecord(record.url) && typeof record.url.path === 'string') continue; + + components[i] = { + id: component.id, + component: 'Loading', + variant: 'block', + }; + replacementCount++; + } + } + + return { messages: cloned, replacementCount }; +} + export async function resolveA2UIImageUrls( messages: A2UIMessage[], ): Promise { const cloned = cloneMessages(messages); const imagePathRefs: ImagePathRef[] = []; + const appendedMessages: A2UIMessage[] = []; - const staticResolutions: Promise[] = []; + const staticResolutions: Promise[] = []; for (const message of cloned) { if (!('updateComponents' in message) || !message.updateComponents) { continue; } const { surfaceId, components } = message.updateComponents; - for (const component of components) { + for (let i = 0; i < components.length; i++) { + const component = components[i]; + if (!component) continue; if (component.component !== 'Image') continue; const record = component as Record; const url = record.url; const fallbackQuery = queryFromComponent(component.id); - if (typeof url === 'string') { + if (typeof url === 'string' && !isLoadableImageSource(url)) { + const originalComponent = { ...component }; + components[i] = createLoadingComponent(component.id) as A2UIComponent; staticResolutions.push( - resolveImageUrl(url, fallbackQuery).then((resolved) => { - record.url = resolved; - }), + resolveImageUrl(url, fallbackQuery).then((resolved) => ({ + version: 'v0.9', + updateComponents: { + surfaceId, + components: [ + { + ...originalComponent, + url: resolved, + }, + ], + }, + })), ); } else if (isRecord(url) && typeof url.path === 'string') { imagePathRefs.push({ @@ -96,9 +168,10 @@ export async function resolveA2UIImageUrls( } } - await Promise.all(staticResolutions); + appendedMessages.push(...await Promise.all(staticResolutions)); - const dataResolutions: Promise[] = []; + const dataResolutions: Promise[] = []; + const pendingImagePathsBySurface = new Map>(); for (const message of cloned) { if (!('updateDataModel' in message) || !message.updateDataModel) { continue; @@ -117,30 +190,117 @@ export async function resolveA2UIImageUrls( const resolvedPaths = new Set(); for (const ref of refs) { const relativePath = relativeJsonPointer(updatePath, ref.path); - resolvedPaths.add(normalizePointer(relativePath)); + resolvedPaths.add(normalizePointer(ref.path)); const current = getAtPointer(dataModel.value, relativePath); + if (isLoadableImageSource(current)) continue; + markPendingImagePath( + pendingImagePathsBySurface, + dataModel.surfaceId, + ref.path, + ); const query = typeof current === 'string' ? current : ref.fallbackQuery; dataResolutions.push( - resolveImageUrl(query, ref.fallbackQuery).then((resolved) => { - dataModel.value = setAtPointer( - dataModel.value, - relativePath, - resolved, - ); - }), + resolveImageUrl(query, ref.fallbackQuery).then((resolved) => ({ + surfaceId: dataModel.surfaceId, + path: normalizePointer(ref.path), + value: resolved, + })), ); } addImageLikeDataResolutions( dataModel.value, - dataModel.value, - '/', + updatePath, + dataModel.surfaceId, resolvedPaths, dataResolutions, ); } - await Promise.all(dataResolutions); - return cloned; + const pendingImageRestores = replacePendingPathImagesWithLoading( + cloned, + pendingImagePathsBySurface, + ); + + const dataPatches = dedupeImageDataPatches( + await Promise.all( + dataResolutions, + ), + ); + appendedMessages.push(...dataPatches.map((patch) => ({ + version: 'v0.9' as const, + updateDataModel: patch, + }))); + appendedMessages.push(...pendingImageRestores); + + return [...cloned, ...appendedMessages]; +} + +function createLoadingComponent(id: string): Record { + return { + id, + component: 'Loading', + variant: 'block', + }; +} + +function markPendingImagePath( + pendingImagePathsBySurface: Map>, + surfaceId: string, + path: string, +): void { + const paths = pendingImagePathsBySurface.get(surfaceId) ?? new Set(); + paths.add(normalizePointer(path)); + pendingImagePathsBySurface.set(surfaceId, paths); +} + +function hasPendingImagePath( + pendingImagePathsBySurface: Map>, + surfaceId: string, + path: string, +): boolean { + return pendingImagePathsBySurface.get(surfaceId)?.has(normalizePointer(path)) + ?? false; +} + +function replacePendingPathImagesWithLoading( + messages: A2UIMessage[], + pendingImagePathsBySurface: Map>, +): A2UIMessage[] { + const restores: A2UIMessage[] = []; + + for (const message of messages) { + if (!('updateComponents' in message) || !message.updateComponents) { + continue; + } + const { surfaceId, components } = message.updateComponents; + for (let i = 0; i < components.length; i++) { + const component = components[i]; + if (!component || component.component !== 'Image') continue; + const url = (component as Record).url; + if (!isRecord(url) || typeof url.path !== 'string') continue; + if ( + !hasPendingImagePath( + pendingImagePathsBySurface, + surfaceId, + url.path, + ) + ) { + continue; + } + + const originalComponent = { ...component }; + components[i] = createLoadingComponent(component.id) as A2UIComponent; + restores.push({ + version: 'v0.9', + updateComponents: { + surfaceId, + components: [originalComponent], + }, + }); + } + } + + return restores; } async function resolveImageUrl( @@ -153,7 +313,9 @@ async function resolveImageUrl( if (!cached) { cached = resolvePexelsImage(query).then( (url) => url ?? picsumUrl(query), - () => picsumUrl(query), + () => fallbackImageUrl(query), + ).catch( + () => fallbackImageUrl(query), ); imageCache.set(cacheKey, cached); } @@ -198,6 +360,21 @@ function picsumUrl(query: string): string { }/1024/768`; } +function fallbackImageUrl(query: string): string { + const label = escapeSvgText(cleanupQuery(query) || 'image unavailable'); + const svg = + `${label}`; + return `data:image/svg+xml,${encodeURIComponent(svg)}`; +} + +function escapeSvgText(value: string): string { + return value + .replace(/&/gu, '&') + .replace(//gu, '>') + .replace(/"/gu, '"'); +} + function normalizeImageQuery(raw: string, fallback: string): string { const trimmed = raw.trim(); if (!trimmed) return fallback; @@ -231,18 +408,18 @@ function queryFromComponent(id: string): string { } function addImageLikeDataResolutions( - root: unknown, value: unknown, path: string, + surfaceId: string, resolvedPaths: Set, - resolutions: Promise[], + resolutions: Promise[], ): void { if (Array.isArray(value)) { value.forEach((item, index) => addImageLikeDataResolutions( - root, item, appendPointer(path, String(index)), + surfaceId, resolvedPaths, resolutions, ) @@ -259,22 +436,25 @@ function addImageLikeDataResolutions( typeof child === 'string' && isImageLikeKey(key) && !resolvedPaths.has(normalizedChildPath) + && !isLoadableImageSource(child) ) { resolvedPaths.add(normalizedChildPath); resolutions.push( resolveImageUrl(child, cleanupQuery(key) || 'image').then( - (resolved) => { - setAtPointer(root, childPath, resolved); - }, + (resolved) => ({ + surfaceId, + path: normalizePointer(childPath), + value: resolved, + }), ), ); continue; } addImageLikeDataResolutions( - root, child, childPath, + surfaceId, resolvedPaths, resolutions, ); @@ -286,6 +466,18 @@ function isImageLikeKey(key: string): boolean { .test(key); } +function dedupeImageDataPatches(patches: ImageDataPatch[]): ImageDataPatch[] { + const seen = new Set(); + const deduped: ImageDataPatch[] = []; + for (const patch of patches) { + const key = `${patch.surfaceId}\0${patch.path}\0${patch.value}`; + if (seen.has(key)) continue; + seen.add(key); + deduped.push(patch); + } + return deduped; +} + function appendPointer(path: string, segment: string): string { const encoded = segment.replace(/~/gu, '~0').replace(/\//gu, '~1'); return path === '/' ? `/${encoded}` : `${path}/${encoded}`; @@ -356,26 +548,3 @@ function getAtPointer(value: unknown, path: string): unknown { } return cursor; } - -function setAtPointer( - value: unknown, - path: string, - nextValue: unknown, -): unknown { - if (path === '/' || path === '') return nextValue; - if (!isRecord(value) && !Array.isArray(value)) return value; - - let cursor = value as Record; - const parts = pointerParts(path); - for (let i = 0; i < parts.length - 1; i++) { - const part = parts[i]; - if (!part) return value; - const child = cursor[part]; - if (!isRecord(child) && !Array.isArray(child)) return value; - cursor = child as Record; - } - - const last = parts[parts.length - 1]; - if (last) cursor[last] = nextValue; - return value; -} diff --git a/packages/genui/server/app/a2ui/_shared.ts b/packages/genui/server/app/a2ui/_shared.ts index 3b2ed4a3ab..56d32d90ee 100644 --- a/packages/genui/server/app/a2ui/_shared.ts +++ b/packages/genui/server/app/a2ui/_shared.ts @@ -20,6 +20,28 @@ export interface A2UIChatBody { validate?: boolean; } +export type A2UIDispatchAction = + | { + event: { + name: string; + context?: Record; + }; + } + | { + functionCall: { + call: string; + args?: Record; + returnType?: string; + }; + }; + +export interface ValidatedAction { + ok: true; + action: A2UIDispatchAction; + kind: 'event' | 'functionCall'; + name: string; +} + function parsePositiveInt( raw: string | undefined, fallback: number, @@ -80,6 +102,64 @@ export function errorMessage( return { message: String(err) }; } +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +export function validateAction(value: unknown): + | ValidatedAction + | { ok: false; status: number; error: string } +{ + if (!isRecord(value)) { + return { + ok: false, + status: 400, + error: 'action.event.name or action.functionCall.call is required', + }; + } + + const hasEvent = 'event' in value; + const hasFunctionCall = 'functionCall' in value; + + if (hasEvent && hasFunctionCall) { + return { + ok: false, + status: 400, + error: 'exactly one of action.event or action.functionCall is required', + }; + } + + if (hasEvent && isRecord(value.event)) { + const name = value.event.name; + if (typeof name === 'string' && name.length > 0) { + return { + ok: true, + action: value as A2UIDispatchAction, + kind: 'event', + name, + }; + } + } + + if (hasFunctionCall && isRecord(value.functionCall)) { + const call = value.functionCall.call; + if (typeof call === 'string' && call.length > 0) { + return { + ok: true, + action: value as A2UIDispatchAction, + kind: 'functionCall', + name: call, + }; + } + } + + return { + ok: false, + status: 400, + error: 'action.event.name or action.functionCall.call is required', + }; +} + export interface ValidatedMessages { ok: true; messages: ChatMessage[]; diff --git a/packages/genui/server/app/a2ui/action/route.ts b/packages/genui/server/app/a2ui/action/route.ts index 6453e3c267..2630ae292b 100644 --- a/packages/genui/server/app/a2ui/action/route.ts +++ b/packages/genui/server/app/a2ui/action/route.ts @@ -9,6 +9,7 @@ import { errorMessage, pickChatOptions, readJsonBodyWithLimit, + validateAction, validateConversation, } from '../_shared'; import { corsPreflight, jsonWithCors } from '../cors'; @@ -20,10 +21,7 @@ export const dynamic = 'force-dynamic'; interface A2UIActionBody { conversation?: unknown; surfaceId?: string; - action: { - name: string; - context?: Record; - }; + action?: unknown; resourceId?: string; model?: string; apiKey?: string; @@ -52,11 +50,12 @@ export async function POST(req: Request) { } const body = parsed.body; - if (!body.action || !body.action.name) { + const validatedAction = validateAction(body.action); + if (!validatedAction.ok) { return jsonWithCors( req, - { ok: false, error: 'action.name is required' }, - { status: 400 }, + { ok: false, error: validatedAction.error }, + { status: validatedAction.status }, ); } @@ -83,7 +82,7 @@ export async function POST(req: Request) { const service = getA2UIAgentService(); const payload = { surfaceId: body.surfaceId, - action: body.action, + action: validatedAction.action, }; const userContent = `A2UI_USER_ACTION: ${JSON.stringify(payload)}`; if (userContent.length > MAX_MESSAGE_CHARS) { diff --git a/packages/genui/server/app/a2ui/action/stream/route.ts b/packages/genui/server/app/a2ui/action/stream/route.ts index ac2d1c6bc5..260e63f300 100644 --- a/packages/genui/server/app/a2ui/action/stream/route.ts +++ b/packages/genui/server/app/a2ui/action/stream/route.ts @@ -12,7 +12,10 @@ import { getA2UIValidationDebugData, validateA2UIOutput, } from '../../../../agent/a2ui-validator'; -import { resolveA2UIImageUrls } from '../../../../agent/image-resolver'; +import { + replacePendingA2UIImagesWithLoading, + resolveA2UIImageUrls, +} from '../../../../agent/image-resolver'; import { getA2UIAgentService } from '../../../../service/a2ui-agent'; import type { ChatMessage } from '../../../../service/a2ui-agent'; import { @@ -20,6 +23,7 @@ import { errorMessage, pickChatOptions, readJsonBodyWithLimit, + validateAction, validateConversation, } from '../../_shared'; import { corsHeaders, corsPreflight, jsonWithCors } from '../../cors'; @@ -54,10 +58,7 @@ function createStreamLogger(route: string) { interface A2UIActionStreamBody { conversation?: unknown; surfaceId?: string; - action: { - name: string; - context?: Record; - }; + action?: unknown; resourceId?: string; model?: string; apiKey?: string; @@ -119,15 +120,16 @@ export async function POST(req: Request) { const body = parsed.body; const validationStartedAt = performance.now(); - if (!body.action || !body.action.name) { + const validatedAction = validateAction(body.action); + if (!validatedAction.ok) { log('action.rejected', { durationMs: performance.now() - validationStartedAt, - error: 'action.name is required', + error: validatedAction.error, }); return jsonWithCors( req, - { ok: false, error: 'action.name is required' }, - { status: 400 }, + { ok: false, error: validatedAction.error }, + { status: validatedAction.status }, ); } @@ -165,7 +167,7 @@ export async function POST(req: Request) { const service = getA2UIAgentService(); const payload = { surfaceId: body.surfaceId, - action: body.action, + action: validatedAction.action, }; const userContent = `A2UI_USER_ACTION: ${JSON.stringify(payload)}`; if (userContent.length > MAX_MESSAGE_CHARS) { @@ -198,7 +200,8 @@ export async function POST(req: Request) { log('request.accepted', { surfaceId: body.surfaceId, - actionName: body.action.name, + actionKind: validatedAction.kind, + actionName: validatedAction.name, conversationHistoryCount: validatedConversation.conversation?.history.length ?? 0, conversationHistoryChars: validatedConversation.conversation?.history @@ -221,6 +224,26 @@ export async function POST(req: Request) { const enqueue = (event: string, data: unknown) => { controller.enqueue(encodeSSE(event, data)); }; + const resolveMessagesForStreaming = async ( + messages: Parameters[0], + ) => { + const pendingImages = replacePendingA2UIImagesWithLoading(messages); + if (pendingImages.replacementCount > 0) { + const loadingMessages = splitA2UIProtocolMessages( + pendingImages.messages, + ); + enqueue('message', { messages: loadingMessages }); + log('images.loading.enqueued', { + replacementCount: pendingImages.replacementCount, + messageCount: loadingMessages.length, + }); + } + + const resolvedMessages = splitA2UIProtocolMessages( + await resolveA2UIImageUrls(messages), + ); + return resolvedMessages; + }; try { const connectStartedAt = performance.now(); @@ -311,7 +334,7 @@ export async function POST(req: Request) { validationOptions, ); let resolvedMessages = v.ok - ? splitA2UIProtocolMessages(await resolveA2UIImageUrls(v.messages)) + ? await resolveMessagesForStreaming(v.messages) : []; log('validation.completed', { ok: v.ok, @@ -357,8 +380,8 @@ export async function POST(req: Request) { finalText = repaired.text; usage = repaired.usage; finishReason = repaired.finishReason; - resolvedMessages = splitA2UIProtocolMessages( - await resolveA2UIImageUrls(repaired.messages), + resolvedMessages = await resolveMessagesForStreaming( + repaired.messages, ); validation = { ok: true, diff --git a/packages/genui/server/app/a2ui/stream/route.ts b/packages/genui/server/app/a2ui/stream/route.ts index 8c8420c093..f2bd9434e2 100644 --- a/packages/genui/server/app/a2ui/stream/route.ts +++ b/packages/genui/server/app/a2ui/stream/route.ts @@ -11,7 +11,10 @@ import { getA2UIValidationDebugData, validateA2UIOutput, } from '../../../agent/a2ui-validator'; -import { resolveA2UIImageUrls } from '../../../agent/image-resolver'; +import { + replacePendingA2UIImagesWithLoading, + resolveA2UIImageUrls, +} from '../../../agent/image-resolver'; import { getA2UIAgentService } from '../../../service/a2ui-agent'; import { errorMessage, @@ -166,6 +169,26 @@ export async function POST(req: Request) { const enqueue = (event: string, data: unknown) => { controller.enqueue(encodeSSE(event, data)); }; + const resolveMessagesForStreaming = async ( + messages: Parameters[0], + ) => { + const pendingImages = replacePendingA2UIImagesWithLoading(messages); + if (pendingImages.replacementCount > 0) { + const loadingMessages = splitA2UIProtocolMessages( + pendingImages.messages, + ); + enqueue('message', { messages: loadingMessages }); + log('images.loading.enqueued', { + replacementCount: pendingImages.replacementCount, + messageCount: loadingMessages.length, + }); + } + + const resolvedMessages = splitA2UIProtocolMessages( + await resolveA2UIImageUrls(messages), + ); + return resolvedMessages; + }; try { const connectStartedAt = performance.now(); @@ -245,7 +268,7 @@ export async function POST(req: Request) { opts.catalog ?? BASIC_CATALOG, ); let resolvedMessages = v.ok - ? splitA2UIProtocolMessages(await resolveA2UIImageUrls(v.messages)) + ? await resolveMessagesForStreaming(v.messages) : []; log('validation.completed', { ok: v.ok, @@ -290,8 +313,8 @@ export async function POST(req: Request) { finalText = repaired.text; usage = repaired.usage; finishReason = repaired.finishReason; - resolvedMessages = splitA2UIProtocolMessages( - await resolveA2UIImageUrls(repaired.messages), + resolvedMessages = await resolveMessagesForStreaming( + repaired.messages, ); validation = { ok: true,