From abda5362cb5a3dd5700d2edca81bee2674dca433 Mon Sep 17 00:00:00 2001 From: Sherry-hue <37186915+Sherry-hue@users.noreply.github.com> Date: Thu, 21 May 2026 16:00:14 +0800 Subject: [PATCH] feat(a2ui): use pexels to generate images --- .../a2ui-playground/src/pages/AIChatPage.tsx | 8 +- packages/genui/server/AGENTS.md | 10 + packages/genui/server/agent/a2ui-prompt.ts | 3 + packages/genui/server/agent/image-resolver.ts | 381 ++++++++++++++++++ .../server/app/a2ui/action/stream/route.ts | 4 +- .../genui/server/app/a2ui/stream/route.ts | 4 +- packages/genui/server/service/a2ui-agent.ts | 4 +- 7 files changed, 409 insertions(+), 5 deletions(-) create mode 100644 packages/genui/server/agent/image-resolver.ts diff --git a/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx b/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx index b20110d868..6686277604 100644 --- a/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx +++ b/packages/genui/a2ui-playground/src/pages/AIChatPage.tsx @@ -337,6 +337,7 @@ async function readA2UIResponse( onText: (text: string) => void, onMessages: (messages: unknown[]) => void, onUsage?: (usage: TokenUsage) => void, + options: { publishPartialMessages?: boolean } = {}, ): Promise { const contentType = response.headers.get('content-type') ?? ''; if (!contentType.includes('text/event-stream')) { @@ -360,6 +361,7 @@ async function readA2UIResponse( let buffer = ''; let generatedText = ''; let latestMessages: unknown[] = []; + const publishPartialMessages = options.publishPartialMessages ?? true; while (true) { const { done, value } = await reader.read(); @@ -376,6 +378,7 @@ async function readA2UIResponse( if (typeof deltaData.text === 'string') { generatedText += deltaData.text; onText(generatedText); + if (!publishPartialMessages) continue; const completed = parseCompletedArrayItems(generatedText); if (completed.length > latestMessages.length) { latestMessages = completed; @@ -417,12 +420,12 @@ const SUGGESTED_PROMPTS: Array<{ label: string; text: string }> = [ { label: '🌤️ Weather with Refresh', text: - 'Create a weather card for San Francisco showing sunny, 22°C, humidity 60%, and a "Refresh" button. When the user taps Refresh, update the card with slightly different weather data to simulate a live fetch.', + 'Create a weather card for San Francisco showing sunny, a photo, 22°C, humidity 60%, and a "Refresh" button. When the user taps Refresh, update the card with slightly different weather data to simulate a live fetch.', }, { label: '🛍️ Product card with Buy', text: - 'Create a product card for a limited-edition sneaker. Include name, price ($189), a short description, and a "Buy Now" button. When tapped, show an order confirmation with a fake order number and estimated delivery.', + 'Create a product card for a limited-edition sneaker. Include name, a photo, price ($189), a short description, and a "Buy Now" button. When tapped, show an order confirmation with a fake order number and estimated delivery.', }, { label: '⚡ Quiz card with actions', @@ -726,6 +729,7 @@ export function AIChatPage( totalTokens: prev.totalTokens + usage.totalTokens, })); }, + { publishPartialMessages: false }, ); if (finalMessages.length === 0) { diff --git a/packages/genui/server/AGENTS.md b/packages/genui/server/AGENTS.md index d9ac1ecd22..4181772d9b 100644 --- a/packages/genui/server/AGENTS.md +++ b/packages/genui/server/AGENTS.md @@ -32,6 +32,16 @@ export OPENAI_MODEL="..." - `OPENAI_BASE_URL` selects the OpenAI-compatible API endpoint. - `OPENAI_MODEL` selects the model used by the A2UI agent. +Image components are resolved after A2UI validation. To enable query-matched +stock images, provide a Pexels API key: + +```bash +export PEXELS_API_KEY="..." +``` + +When `PEXELS_API_KEY` is absent or Pexels returns no result, the server falls +back to a deterministic Picsum URL. + The server fails fast at startup (via `instrumentation.ts`) when any of these are missing in production. In development, a warning is logged instead so the playground keeps working. diff --git a/packages/genui/server/agent/a2ui-prompt.ts b/packages/genui/server/agent/a2ui-prompt.ts index 6b522307ab..5edde4a27c 100644 --- a/packages/genui/server/agent/a2ui-prompt.ts +++ b/packages/genui/server/agent/a2ui-prompt.ts @@ -123,6 +123,9 @@ function buildHardRules(catalogId: string): string { 16. For UI that should change after a button tap, keep the initial response in the pre-action state. Put confirmation, success, or result details in the action response instead of showing them before the action happens. +17. For Image.url, provide a short English image search query such as + "fresh pasta on a table" or "city skyline at night". Do NOT invent photo + CDN URLs. The server resolves Image.url values through its image provider. `; } diff --git a/packages/genui/server/agent/image-resolver.ts b/packages/genui/server/agent/image-resolver.ts new file mode 100644 index 0000000000..bf98ac0ed6 --- /dev/null +++ b/packages/genui/server/agent/image-resolver.ts @@ -0,0 +1,381 @@ +// Copyright 2026 The Lynx Authors. All rights reserved. +// Licensed under the Apache License Version 2.0 that can be found in the +// LICENSE file in the root directory of this source tree. + +import type { A2UIMessage } from './a2ui-validator'; + +interface ImagePathRef { + surfaceId: string; + path: string; + fallbackQuery: string; +} + +interface PexelsPhoto { + src?: { + large2x?: string; + large?: string; + medium?: string; + original?: string; + }; +} + +interface PexelsSearchResponse { + photos?: PexelsPhoto[]; +} + +const IMAGE_CACHE_MAX_ENTRIES = readPositiveIntegerEnv( + 'A2UI_IMAGE_CACHE_MAX_ENTRIES', + 1000, +); +const PEXELS_REQUEST_TIMEOUT_MS = readPositiveIntegerEnv( + 'A2UI_PEXELS_REQUEST_TIMEOUT_MS', + 5000, +); + +class LruCache { + private readonly entries = new Map(); + + public constructor(private readonly maxEntries: number) {} + + public get(key: K): V | undefined { + const value = this.entries.get(key); + if (value === undefined) return undefined; + this.entries.delete(key); + this.entries.set(key, value); + return value; + } + + public set(key: K, value: V): void { + if (this.entries.has(key)) { + this.entries.delete(key); + } + this.entries.set(key, value); + + while (this.entries.size > this.maxEntries) { + const oldestKey = this.entries.keys().next().value; + if (oldestKey === undefined) break; + this.entries.delete(oldestKey); + } + } +} + +const imageCache = new LruCache>( + IMAGE_CACHE_MAX_ENTRIES, +); + +export async function resolveA2UIImageUrls( + messages: A2UIMessage[], +): Promise { + const cloned = cloneMessages(messages); + const imagePathRefs: ImagePathRef[] = []; + + const staticResolutions: Promise[] = []; + for (const message of cloned) { + if (!('updateComponents' in message) || !message.updateComponents) { + continue; + } + const { surfaceId, components } = message.updateComponents; + for (const component of components) { + if (component.component !== 'Image') continue; + const record = component as Record; + const url = record.url; + const fallbackQuery = queryFromComponent(component.id); + if (typeof url === 'string') { + staticResolutions.push( + resolveImageUrl(url, fallbackQuery).then((resolved) => { + record.url = resolved; + }), + ); + } else if (isRecord(url) && typeof url.path === 'string') { + imagePathRefs.push({ + surfaceId, + path: url.path, + fallbackQuery, + }); + } + } + } + + await Promise.all(staticResolutions); + + const dataResolutions: Promise[] = []; + for (const message of cloned) { + if (!('updateDataModel' in message) || !message.updateDataModel) { + continue; + } + const dataModel = message.updateDataModel as + & typeof message.updateDataModel + & { value?: unknown }; + if (!('value' in dataModel)) continue; + + const updatePath = dataModel.path ?? '/'; + const refs = imagePathRefs.filter( + (ref) => + ref.surfaceId === dataModel.surfaceId + && pathContains(updatePath, ref.path), + ); + const resolvedPaths = new Set(); + for (const ref of refs) { + const relativePath = relativeJsonPointer(updatePath, ref.path); + resolvedPaths.add(normalizePointer(relativePath)); + const current = getAtPointer(dataModel.value, relativePath); + const query = typeof current === 'string' ? current : ref.fallbackQuery; + dataResolutions.push( + resolveImageUrl(query, ref.fallbackQuery).then((resolved) => { + dataModel.value = setAtPointer( + dataModel.value, + relativePath, + resolved, + ); + }), + ); + } + addImageLikeDataResolutions( + dataModel.value, + dataModel.value, + '/', + resolvedPaths, + dataResolutions, + ); + } + + await Promise.all(dataResolutions); + return cloned; +} + +async function resolveImageUrl( + rawQuery: string, + fallbackQuery: string, +): Promise { + const query = normalizeImageQuery(rawQuery, fallbackQuery); + const cacheKey = query.toLowerCase(); + let cached = imageCache.get(cacheKey); + if (!cached) { + cached = resolvePexelsImage(query).then( + (url) => url ?? picsumUrl(query), + () => picsumUrl(query), + ); + imageCache.set(cacheKey, cached); + } + return cached; +} + +async function resolvePexelsImage(query: string): Promise { + const apiKey = process.env.PEXELS_API_KEY; + if (!apiKey) return null; + + const url = new URL('https://api.pexels.com/v1/search'); + url.searchParams.set('query', query); + url.searchParams.set('per_page', '1'); + url.searchParams.set('orientation', 'landscape'); + + const controller = new AbortController(); + const timeout = setTimeout( + () => controller.abort(), + PEXELS_REQUEST_TIMEOUT_MS, + ); + + try { + const res = await fetch(url, { + headers: { Authorization: apiKey }, + signal: controller.signal, + }); + if (!res.ok) return null; + + const data = await res.json() as PexelsSearchResponse; + const src = data.photos?.[0]?.src; + return src?.large2x ?? src?.large ?? src?.medium ?? src?.original ?? null; + } catch { + return null; + } finally { + clearTimeout(timeout); + } +} + +function picsumUrl(query: string): string { + return `https://picsum.photos/seed/${ + encodeURIComponent(hashSeed(query)) + }/1024/768`; +} + +function normalizeImageQuery(raw: string, fallback: string): string { + const trimmed = raw.trim(); + if (!trimmed) return fallback; + + try { + const url = new URL(trimmed); + const tokens = [ + url.hostname.replace(/\.(?:com|org|net|cn)$/u, ''), + ...url.pathname.split('/'), + ]; + const query = tokens.join(' '); + return cleanupQuery(query) || fallback; + } catch { + return cleanupQuery(trimmed) || fallback; + } +} + +function cleanupQuery(value: string): string { + return value + .replace(/https?:\/\//giu, ' ') + .replace(/\.(?:jpg|jpeg|png|webp|gif|avif)\b/giu, ' ') + .replace(/[_\-./?=&%]+/gu, ' ') + .replace(/\b\d{2,}\b/gu, ' ') + .replace(/\s+/gu, ' ') + .trim() + .slice(0, 80); +} + +function queryFromComponent(id: string): string { + return cleanupQuery(id) || 'app interface illustration'; +} + +function addImageLikeDataResolutions( + root: unknown, + value: unknown, + path: string, + resolvedPaths: Set, + resolutions: Promise[], +): void { + if (Array.isArray(value)) { + value.forEach((item, index) => + addImageLikeDataResolutions( + root, + item, + appendPointer(path, String(index)), + resolvedPaths, + resolutions, + ) + ); + return; + } + + if (!isRecord(value)) return; + + for (const [key, child] of Object.entries(value)) { + const childPath = appendPointer(path, key); + const normalizedChildPath = normalizePointer(childPath); + if ( + typeof child === 'string' + && isImageLikeKey(key) + && !resolvedPaths.has(normalizedChildPath) + ) { + resolvedPaths.add(normalizedChildPath); + resolutions.push( + resolveImageUrl(child, cleanupQuery(key) || 'image').then( + (resolved) => { + setAtPointer(root, childPath, resolved); + }, + ), + ); + continue; + } + + addImageLikeDataResolutions( + root, + child, + childPath, + resolvedPaths, + resolutions, + ); + } +} + +function isImageLikeKey(key: string): boolean { + return /(?:^|[-_])(?:image|photo|picture|avatar|cover|poster|artwork|thumbnail)(?:$|[-_])/iu + .test(key); +} + +function appendPointer(path: string, segment: string): string { + const encoded = segment.replace(/~/gu, '~0').replace(/\//gu, '~1'); + return path === '/' ? `/${encoded}` : `${path}/${encoded}`; +} + +function hashSeed(value: string): string { + let hash = 0; + for (let i = 0; i < value.length; i++) { + hash = Math.imul(31, hash) + value.charCodeAt(i) | 0; + } + return `a2ui-${Math.abs(hash).toString(36)}`; +} + +function readPositiveIntegerEnv(name: string, fallback: number): number { + const raw = process.env[name]; + if (!raw) return fallback; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function cloneMessages(messages: A2UIMessage[]): A2UIMessage[] { + return JSON.parse(JSON.stringify(messages)) as A2UIMessage[]; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function normalizePointer(path: string): string { + if (!path || path === '/') return '/'; + return path.startsWith('/') ? path : `/${path}`; +} + +function pathContains(parent: string, child: string): boolean { + const normalizedParent = normalizePointer(parent); + const normalizedChild = normalizePointer(child); + return normalizedParent === '/' + || normalizedParent === normalizedChild + || normalizedChild.startsWith(`${normalizedParent}/`); +} + +function relativeJsonPointer(parent: string, child: string): string { + const normalizedParent = normalizePointer(parent); + const normalizedChild = normalizePointer(child); + if (normalizedParent === '/') return normalizedChild; + if (normalizedParent === normalizedChild) return '/'; + return normalizedChild.slice(normalizedParent.length) || '/'; +} + +function decodePointerSegment(segment: string): string { + return segment.replace(/~1/gu, '/').replace(/~0/gu, '~'); +} + +function pointerParts(path: string): string[] { + return normalizePointer(path) + .split('/') + .slice(1) + .filter(Boolean) + .map((segment) => decodePointerSegment(segment)); +} + +function getAtPointer(value: unknown, path: string): unknown { + if (path === '/' || path === '') return value; + let cursor = value; + for (const part of pointerParts(path)) { + if (!isRecord(cursor) && !Array.isArray(cursor)) return undefined; + cursor = (cursor as Record)[part]; + } + return cursor; +} + +function setAtPointer( + value: unknown, + path: string, + nextValue: unknown, +): unknown { + if (path === '/' || path === '') return nextValue; + if (!isRecord(value) && !Array.isArray(value)) return value; + + let cursor = value as Record; + const parts = pointerParts(path); + for (let i = 0; i < parts.length - 1; i++) { + const part = parts[i]; + if (!part) return value; + const child = cursor[part]; + if (!isRecord(child) && !Array.isArray(child)) return value; + cursor = child as Record; + } + + const last = parts[parts.length - 1]; + if (last) cursor[last] = nextValue; + return value; +} diff --git a/packages/genui/server/app/a2ui/action/stream/route.ts b/packages/genui/server/app/a2ui/action/stream/route.ts index 359d7ba7f3..811a30eea0 100644 --- a/packages/genui/server/app/a2ui/action/stream/route.ts +++ b/packages/genui/server/app/a2ui/action/stream/route.ts @@ -5,6 +5,7 @@ import type { A2UICatalog } from '../../../../agent/a2ui-catalog'; import { BASIC_CATALOG } from '../../../../agent/a2ui-catalog'; import { validateA2UIOutput } from '../../../../agent/a2ui-validator'; +import { resolveA2UIImageUrls } from '../../../../agent/image-resolver'; import { getA2UIAgentService } from '../../../../service/a2ui-agent'; import type { ChatMessage } from '../../../../service/a2ui-agent'; import { @@ -155,10 +156,11 @@ export async function POST(req: Request) { existingSurfaceIds: body.surfaceId ? [body.surfaceId] : [], }, ); + const messages = v.ok ? await resolveA2UIImageUrls(v.messages) : []; validation = { ok: v.ok, errors: v.errors, - messages: v.ok ? v.messages : [], + messages, }; } diff --git a/packages/genui/server/app/a2ui/stream/route.ts b/packages/genui/server/app/a2ui/stream/route.ts index a908ee2bd0..b47231dee4 100644 --- a/packages/genui/server/app/a2ui/stream/route.ts +++ b/packages/genui/server/app/a2ui/stream/route.ts @@ -4,6 +4,7 @@ import { BASIC_CATALOG } from '../../../agent/a2ui-catalog'; import { validateA2UIOutput } from '../../../agent/a2ui-validator'; +import { resolveA2UIImageUrls } from '../../../agent/image-resolver'; import { getA2UIAgentService } from '../../../service/a2ui-agent'; import { errorMessage, @@ -103,10 +104,11 @@ export async function POST(req: Request) { finalText, opts.catalog ?? BASIC_CATALOG, ); + const messages = v.ok ? await resolveA2UIImageUrls(v.messages) : []; validation = { ok: v.ok, errors: v.errors, - messages: v.ok ? v.messages : [], + messages, }; } diff --git a/packages/genui/server/service/a2ui-agent.ts b/packages/genui/server/service/a2ui-agent.ts index 85d79bcd93..089846dd20 100644 --- a/packages/genui/server/service/a2ui-agent.ts +++ b/packages/genui/server/service/a2ui-agent.ts @@ -13,6 +13,7 @@ import { validateA2UIOutput, } from '../agent/a2ui-validator'; import type { A2UIMessage, ValidationOptions } from '../agent/a2ui-validator'; +import { resolveA2UIImageUrls } from '../agent/image-resolver'; export interface ChatMessage { role: 'user' | 'assistant' | 'system'; @@ -256,10 +257,11 @@ export default class A2UIAgentService { const validation = validateA2UIOutput(text, catalog, validationOptions); if (validation.ok) { + const messages = await resolveA2UIImageUrls(validation.messages); return { ok: true, text, - messages: validation.messages, + messages, errors: [], attempts: attempt, usage: lastUsage,