From eddbe6106f3db4c2cc6c5a698ca5e4562b42bb15 Mon Sep 17 00:00:00 2001 From: Sherry-hue <37186915+Sherry-hue@users.noreply.github.com> Date: Tue, 19 May 2026 14:21:02 +0800 Subject: [PATCH] feat(a2ui): chat add action request and response --- .../a2ui-playground/lynx-src/a2ui/App.tsx | 28 + .../a2ui-playground/src/pages/AIChatPage.css | 251 ++++++++- .../a2ui-playground/src/pages/AIChatPage.tsx | 501 ++++++++++++++++-- packages/genui/a2ui-playground/src/render.tsx | 63 ++- .../a2ui-playground/src/utils/renderUrl.ts | 6 + packages/genui/server/agent/a2ui-catalog.ts | 15 +- packages/genui/server/agent/a2ui-prompt.ts | 26 +- packages/genui/server/agent/a2ui-validator.ts | 10 +- .../server/app/a2ui/action/stream/route.ts | 189 +++++++ .../genui/server/app/a2ui/stream/route.ts | 18 +- packages/genui/server/service/a2ui-agent.ts | 10 + 11 files changed, 1038 insertions(+), 79 deletions(-) create mode 100644 packages/genui/server/app/a2ui/action/stream/route.ts diff --git a/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx b/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx index 8fd121b57a..f8bcf6186c 100644 --- a/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx +++ b/packages/genui/a2ui-playground/lynx-src/a2ui/App.tsx @@ -106,6 +106,7 @@ interface InitData { playbackMode?: boolean; theme?: 'light' | 'dark'; playbackPaused?: boolean; + liveAction?: boolean; } type Theme = 'light' | 'dark'; @@ -211,6 +212,12 @@ function normalizeInitDataLike(raw: unknown): InitData { out.theme = theme.toLowerCase() as Theme; } + const liveAction = obj.liveAction; + if (liveAction !== undefined) { + out.liveAction = liveAction === true || liveAction === '1' + || liveAction === 1; + } + return out; } @@ -224,6 +231,7 @@ function mergeInitDataPreferLeft(a: InitData, b: InitData): InitData { playbackMode: a.playbackMode ?? b.playbackMode, playbackPaused: a.playbackPaused ?? b.playbackPaused, theme: a.theme ?? b.theme, + liveAction: a.liveAction ?? b.liveAction, }; } @@ -414,6 +422,18 @@ export function App() { }, ); + useLynxGlobalEventListener( + 'A2UI_ACTION_RESPONSE', + (messages: unknown) => { + const currentStore = storeRef.current; + if (!currentStore) return; + const normalized = normalizeProtocolMessages(messages); + for (const msg of normalized) { + currentStore.push(msg); + } + }, + ); + useEffect(() => { playbackPausedRef.current = isPlaybackPaused; }, [isPlaybackPaused]); @@ -516,6 +536,14 @@ export function App() { messageStore={store} catalogs={ALL_BUILTINS} onAction={(action) => { + if (effectiveData.liveAction) { + NativeModules.bridge.call( + 'A2UI_USER_ACTION', + action as unknown as Record, + () => undefined, + ); + return; + } // Forward user actions to the mock agent — it pushes // the canned response messages back into the same store. void agentRef.current?.onAction(action); diff --git a/packages/genui/a2ui-playground/src/pages/AIChatPage.css b/packages/genui/a2ui-playground/src/pages/AIChatPage.css index c23e306c93..c0e5cf16f1 100644 --- a/packages/genui/a2ui-playground/src/pages/AIChatPage.css +++ b/packages/genui/a2ui-playground/src/pages/AIChatPage.css @@ -17,6 +17,7 @@ display: flex; flex-direction: column; min-width: 0; + min-height: 0; } .chatHeader { @@ -62,6 +63,7 @@ .chatMessages { flex: 1; + min-height: 0; overflow-y: auto; padding: 20px; display: flex; @@ -76,6 +78,7 @@ font-size: 14px; line-height: 1.5; word-break: break-word; + flex-shrink: 0; } .chatMessageUser { @@ -92,14 +95,202 @@ border-bottom-left-radius: 4px; } -.chatGeneratedJson { +.chatMessageAction { + align-self: center; + max-width: 90%; + padding: 6px 12px; + background: transparent; + border: 1px dashed var(--geist-border); + border-radius: 999px; + color: var(--geist-secondary); + font-size: 12px; + font-family: var(--geist-mono); +} + +.chatMessageActionPending { + display: inline-flex; + align-items: center; + gap: 8px; +} + +.chatMessageActionSpinner { + width: 12px; + height: 12px; + border: 1.5px solid + color-mix(in srgb, var(--geist-secondary) 35%, transparent); + border-top-color: var(--geist-secondary); + border-radius: 50%; + display: inline-block; + animation: chat-message-action-spin 0.8s linear infinite; + flex-shrink: 0; +} + +@keyframes chat-message-action-spin { + to { + transform: rotate(360deg); + } +} + +.chatMessageAction.chatMessageActionExpanded { + align-self: flex-start; width: 100%; - max-width: 100%; + max-width: 80%; + padding: 0; + border: 1px solid var(--geist-border); + border-radius: var(--geist-radius-md); + background: var(--geist-surface); + overflow: hidden; +} + +.chatMessageStatus { align-self: stretch; + display: flex; + align-items: center; + gap: 8px; + max-width: 100%; + padding: 6px 10px; + border: none; + border-radius: var(--geist-radius-md); + background: transparent; + color: var(--geist-secondary); + font-size: 12px; + font-family: var(--geist-mono); + line-height: 1.5; +} + +.chatMessageStatus .chatMessageBody { + display: inline-flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.chatMessageStatusIcon { + font-size: 14px; + line-height: 1; + flex-shrink: 0; +} + +.chatMessageStatusInline { + padding: 1px 6px; + border: 1px solid var(--geist-border); + border-radius: 4px; + background: var(--geist-surface); + color: var(--geist-foreground); + font-family: var(--geist-mono); + font-size: 11px; +} + +.chatMessageStatus-info { + color: var(--geist-secondary); +} + +.chatMessageStatus-pending { + color: var(--geist-foreground); +} + +.chatMessageStatus-success { + color: color-mix( + in srgb, + var(--geist-success, #0a7d2c) 92%, + var(--geist-foreground) + ); +} + +.chatMessageStatus-error { + color: color-mix( + in srgb, + var(--geist-error, #c0392b) 92%, + var(--geist-foreground) + ); +} + +.chatMessageAction.chatMessageActionExpanded .chatMessageBody { + padding: 8px 12px; + border-bottom: 1px solid var(--geist-border); + background: var(--geist-background); + color: var(--geist-foreground); +} + +.chatMessageJson { + align-self: flex-start; + width: 100%; + max-width: 80%; + padding: 0; border: 1px solid var(--geist-border); border-radius: var(--geist-radius-md); + background: var(--geist-surface); overflow: hidden; - background: var(--geist-code-bg); +} + +.chatMessageJson .chatMessageBody { + display: flex; + align-items: center; + min-height: 34px; + padding: 0 12px; + border-bottom: 1px solid var(--geist-border); + background: var(--geist-background); + color: var(--geist-secondary); + font-size: 11px; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.chatMessagePayload { + display: flex; + flex-direction: column; +} + +.chatMessagePayloadLabel { + padding: 6px 12px; + border-bottom: 1px solid var(--geist-border); + background: var(--geist-background); + color: var(--geist-secondary); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.chatMessagePayloadEditor .cm-editor { + height: auto; + max-height: 320px; + font-family: var(--geist-mono); + font-size: 12px; + background: var(--geist-surface); + color: var(--geist-foreground); +} + +.chatMessagePayloadEditor .cm-scroller { + max-height: 320px; + overflow: auto; +} + +.chatMessagePayloadEditor .cm-gutters { + background: color-mix(in srgb, var(--geist-surface) 88%, var(--geist-border)); + color: var(--geist-secondary); + border-right: 1px solid + color-mix(in srgb, var(--geist-border) 82%, transparent); +} + +.chatMessagePayloadEditor .cm-content { + caret-color: var(--geist-foreground); +} + +.chatMessagePayloadEditor .cm-activeLine, +.chatMessagePayloadEditor .cm-activeLineGutter { + background: color-mix(in srgb, var(--geist-foreground) 4%, transparent); +} + +.chatGeneratedJson { + width: 100%; + max-width: 80%; + align-self: flex-start; + border: 1px solid var(--geist-border); + border-radius: var(--geist-radius-md); + overflow: hidden; + background: var(--geist-surface); flex-shrink: 0; } @@ -131,28 +322,28 @@ .chatGeneratedJsonEditor .cm-editor { height: auto; - max-height: 400px; + max-height: 480px; font-family: var(--geist-mono); font-size: 13px; - background: var(--geist-code-bg); - color: var(--geist-code-fg); + background: var(--geist-surface); + color: var(--geist-foreground); } .chatGeneratedJsonEditor .cm-scroller { - max-height: 400px; + max-height: 480px; overflow: auto; } .chatGeneratedJsonEditor .cm-gutters { - background: color-mix(in srgb, var(--geist-code-bg) 88%, black); - color: color-mix(in srgb, var(--geist-code-fg) 58%, white); + background: color-mix(in srgb, var(--geist-surface) 88%, var(--geist-border)); + color: var(--geist-secondary); border-right: 1px solid color-mix(in srgb, var(--geist-border) 82%, transparent); } .chatGeneratedJsonEditor .cm-activeLine, .chatGeneratedJsonEditor .cm-activeLineGutter { - background: color-mix(in srgb, var(--geist-background) 12%, transparent); + background: color-mix(in srgb, var(--geist-foreground) 4%, transparent); } .chatInputArea { @@ -160,10 +351,50 @@ border-top: 1px solid var(--geist-border); background: var(--geist-background); display: flex; + flex-direction: column; gap: 8px; flex-shrink: 0; } +.chatSuggestionsRow { + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.chatSuggestionChip { + display: inline-flex; + align-items: center; + height: 28px; + padding: 0 12px; + border: 1px solid var(--geist-border); + border-radius: 999px; + background: var(--geist-surface); + color: var(--geist-foreground); + font-size: 12px; + font-weight: 500; + cursor: pointer; + transition: + background var(--geist-transition), + border-color var(--geist-transition); + white-space: nowrap; +} + +.chatSuggestionChip:hover { + background: color-mix(in srgb, var(--geist-surface) 80%, var(--geist-border)); + border-color: var(--geist-foreground); +} + +.chatSuggestionChip:disabled { + opacity: 0.45; + cursor: not-allowed; +} + +.chatInputRow { + display: flex; + gap: 8px; +} + .chatInput { flex: 1; height: 40px; diff --git a/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx b/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx index a678675b16..95b6e1bded 100644 --- a/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx +++ b/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx @@ -2,7 +2,7 @@ // Licensed under the Apache License Version 2.0 that can be found in the // LICENSE file in the root directory of this source tree. import { json } from '@codemirror/lang-json'; -import CodeMirror from '@uiw/react-codemirror'; +import CodeMirror, { EditorView } from '@uiw/react-codemirror'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import './AIChatPage.css'; @@ -16,8 +16,11 @@ import type { Protocol } from '../utils/protocol.js'; import { buildRenderUrl } from '../utils/renderUrl.js'; interface ChatMessage { - role: 'user' | 'ai'; + role: 'user' | 'ai' | 'action' | 'json' | 'status'; content: string | React.ReactNode; + payload?: unknown; + payloadLabel?: string; + tone?: 'info' | 'pending' | 'success' | 'error'; } interface ModelChatMessage { @@ -72,7 +75,7 @@ const RESIZE_BREAKPOINT = 980; const ONLINE_A2UI_SERVER_ORIGIN = 'https://genui-server.vercel.app'; const ONLINE_A2UI_CHAT_URL = `${ONLINE_A2UI_SERVER_ORIGIN}/a2ui/stream`; const LOCAL_A2UI_SERVER_PORT = '3060'; -const jsonExtensions = [json()]; +const jsonExtensions = [json(), EditorView.lineWrapping]; function isDevHost(hostname: string): boolean { return ( @@ -124,6 +127,13 @@ function getA2UIChatEndpoint(): string { return ONLINE_A2UI_CHAT_URL; } +function getA2UIActionStreamEndpoint(): string { + return getA2UIChatEndpoint().replace( + /\/a2ui\/stream$/, + '/a2ui/action/stream', + ); +} + function getErrorMessage(error: unknown): string { if (error instanceof Error) return error.message; return String(error); @@ -161,6 +171,23 @@ function parseSseData(raw: string): unknown { } } +function safeStringifyPayload(value: unknown): string { + if (typeof value === 'string') { + // Streaming JSON often arrives minified (no spaces/newlines) — try to + // re-pretty-print it so CodeMirror can show it across multiple lines. + try { + return JSON.stringify(JSON.parse(value), null, 2); + } catch { + return value; + } + } + try { + return JSON.stringify(value, null, 2); + } catch { + return String(value); + } +} + function parseSseFrame(frame: string): SseEvent | null { const lines = frame.split(/\r?\n/u); let event = 'message'; @@ -266,6 +293,7 @@ async function readA2UIResponse( response: BrowserResponse, onText: (text: string) => void, onMessages: (messages: unknown[]) => void, + onStart?: (threadId: string) => void, ): Promise { const contentType = response.headers.get('content-type') ?? ''; if (!contentType.includes('text/event-stream')) { @@ -296,10 +324,18 @@ async function readA2UIResponse( const parsed = parseSseFrame(frame); if (!parsed) continue; + if (parsed.event === 'start') { + const data = parsed.data as { threadId?: unknown }; + if (typeof data.threadId === 'string') { + onStart?.(data.threadId); + } + continue; + } + if (parsed.event === 'delta') { - const data = parsed.data as { text?: unknown }; - if (typeof data.text === 'string') { - generatedText += data.text; + const deltaData = parsed.data as { text?: unknown }; + if (typeof deltaData.text === 'string') { + generatedText += deltaData.text; onText(generatedText); const completed = parseCompletedArrayItems(generatedText); if (completed.length > latestMessages.length) { @@ -334,6 +370,24 @@ async function readA2UIResponse( : normalizeA2UIMessages(generatedText); } +const SUGGESTED_PROMPTS: Array<{ label: string; text: string }> = [ + { + label: '⚡ Quiz card with actions', + text: + 'Create a trivia quiz card. Show a question "What is the capital of Japan?" with 4 answer buttons: Tokyo, Beijing, Seoul, Bangkok. When the user taps an answer, show whether it is correct with a brief explanation.', + }, + { + label: '🛍️ Product card with Buy', + text: + 'Create a product card for a limited-edition sneaker. Include name, price ($189), a short description, and a "Buy Now" button. When tapped, show an order confirmation with a fake order number and estimated delivery.', + }, + { + label: '🌤️ Weather with Refresh', + text: + 'Create a weather card for San Francisco showing sunny, 22°C, humidity 60%, and a "Refresh" button. When the user taps Refresh, update the card with slightly different weather data to simulate a live fetch.', + }, +]; + export function AIChatPage(props: { protocol: Protocol }) { const { protocol } = props; const [messages, setMessages] = useState([WELCOME_MESSAGE]); @@ -345,10 +399,14 @@ export function AIChatPage(props: { protocol: Protocol }) { ); const [isGenerating, setIsGenerating] = useState(false); const messagesEndRef = useRef(null); + const chatMessagesRef = useRef(null); + const followBottomRef = useRef(true); const previewFrameRef = useRef(null); const abortRef = useRef(null); + const actionAbortRef = useRef(null); const conversationRef = useRef([]); const latestPreviewMessagesRef = useRef([]); + const threadIdRef = useRef(null); const { containerRef: pageRef, handleResizeStart: handlePanelResizeStart, @@ -367,8 +425,69 @@ export function AIChatPage(props: { protocol: Protocol }) { }); useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); - }); + // Re-run on every render so streaming text growth & async editor mounts + // both keep the chat pinned to the latest message. + void messages; + void generatedJson; + void isGenerating; + if (!followBottomRef.current) return; + const container = chatMessagesRef.current; + if (!container) return; + container.scrollTop = container.scrollHeight; + }, [messages, generatedJson, isGenerating]); + + useEffect(() => { + const container = chatMessagesRef.current; + if (!container) return; + if (typeof ResizeObserver === 'undefined') return; + // Async-mounted CodeMirror editors and streaming JSON expand the container + // height after React commits. ResizeObserver fires for those layout shifts + // and lets us keep the chat pinned to the bottom while the user is in + // "follow" mode. + const sizeObserver = new ResizeObserver(() => { + if (!followBottomRef.current) return; + container.scrollTop = container.scrollHeight; + }); + sizeObserver.observe(container); + Array.from(container.children).forEach((child: Element) => { + sizeObserver.observe(child); + }); + // Newly inserted message rows must also be observed so their delayed + // CodeMirror layout still triggers the bottom-pin behavior. We use a + // MutationObserver instead of re-running this effect on every messages + // change so it stays a one-time setup. + const childObserver = new MutationObserver((entries) => { + let inserted = false; + for (const entry of entries) { + entry.addedNodes.forEach((node) => { + if (node instanceof Element) { + sizeObserver.observe(node); + inserted = true; + } + }); + } + if (inserted && followBottomRef.current) { + container.scrollTop = container.scrollHeight; + } + }); + childObserver.observe(container, { childList: true }); + return () => { + sizeObserver.disconnect(); + childObserver.disconnect(); + }; + }, []); + + const handleChatScroll = useCallback(() => { + const container = chatMessagesRef.current; + if (!container) return; + const distanceFromBottom = container.scrollHeight + - container.scrollTop + - container.clientHeight; + // 32px hysteresis: small upward scrolls inside CodeMirror still count as + // "at bottom"; once the user is clearly above we stop auto-following so + // their reading position is respected. + followBottomRef.current = distanceFromBottom <= 32; + }, []); const baseUrl = useMemo(() => window.location.href.replace(/#.*$/, ''), []); const previewSource = useMemo(() => { @@ -393,6 +512,7 @@ export function AIChatPage(props: { protocol: Protocol }) { demoUrl: DEFAULT_A2UI_DEMO_URL, messages: nextMessages, instant: true, + liveAction: !!threadIdRef.current, }; setRenderUrl((current) => { @@ -443,6 +563,7 @@ export function AIChatPage(props: { protocol: Protocol }) { setGeneratedJson(''); setPreviewMessages(null); latestPreviewMessagesRef.current = []; + threadIdRef.current = null; setIsGenerating(true); void (async () => { @@ -474,6 +595,9 @@ export function AIChatPage(props: { protocol: Protocol }) { }); }, publishPreviewMessages, + (threadId) => { + threadIdRef.current = threadId; + }, ); if (finalMessages.length === 0) { @@ -495,6 +619,12 @@ export function AIChatPage(props: { protocol: Protocol }) { finalMessages.length === 1 ? '' : 's' }.`, }; + next.push({ + role: 'json', + content: 'Generated Output', + payload: assistantContent, + payloadLabel: 'JSON', + }); return next; }); } catch (e) { @@ -517,6 +647,236 @@ export function AIChatPage(props: { protocol: Protocol }) { })(); }, [inputValue, isGenerating, publishPreviewMessages]); + useEffect(() => { + const handleMessage = (e: MessageEvent) => { + if (!e.data || typeof e.data !== 'object') return; + const msg = e.data as Record; + if (msg.type !== 'A2UI_USER_ACTION') return; + + const threadId = threadIdRef.current; + if (!threadId) return; + + const action = msg.action as { + name?: string; + context?: Record; + }; + const actionName = typeof action?.name === 'string' + ? action.name + : 'unknown'; + + // Abort any in-flight action stream so a stale request can no longer + // mutate state or post into the preview iframe after a new action + // arrives. + actionAbortRef.current?.abort(); + const controller = new AbortController(); + actionAbortRef.current = controller; + const signal = controller.signal; + + let pendingIndex = -1; + let streamingIndex = -1; + setMessages((prev) => { + const next: ChatMessage[] = [ + ...prev, + { + role: 'status' as const, + tone: 'info', + content: ( + <> + + + Lynx Preview triggered{' '} + {actionName} + , forwarding request to agent... + + + ), + }, + { + role: 'action' as const, + content: `⚡ Action: ${actionName}`, + payload: action, + payloadLabel: 'REQUEST', + }, + { + role: 'status' as const, + tone: 'pending', + content: ( + <> +