Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions studio/frontend/src/components/assistant-ui/thread.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ import {
DownloadIcon,
GlobeIcon,
HeadphonesIcon,
ImageIcon,
LightbulbIcon,
LightbulbOffIcon,
MicIcon,
Expand All @@ -89,6 +88,7 @@ import {
Copy01Icon,
Delete02Icon,
Edit03Icon,
Image03Icon,
Tick02Icon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
Expand Down Expand Up @@ -271,17 +271,17 @@ const GeneratedImageViewportOverlay: FC<{ hideComposer?: boolean }> = ({
className="w-full max-w-[min(100%,46rem)] shrink-0 text-center"
title={overlay.title}
>
<p className="truncate font-medium text-foreground/70 text-xs">
<p className="truncate text-xs font-semibold text-foreground/80">
Generated image
</p>
{overlay.metadata ? (
<p className="truncate text-[11px] text-muted-foreground/75">
<p className="truncate text-[11px] font-medium text-muted-foreground">
{overlay.metadata}
</p>
) : null}
{hideComposer ? null : (
<p className="mt-1 text-muted-foreground text-xs">
Type edits below, then send.
<p className="mx-auto mt-2 inline-flex rounded-full bg-primary/10 px-3 py-1 text-xs font-medium text-primary">
Type edits below, then send
</p>
)}
</div>
Expand Down Expand Up @@ -525,13 +525,15 @@ const Composer: FC<{
<PendingAudioChip />
<ToolStatusDisplay />
<ComposerPrimitive.Input
placeholder="Send a message..."
placeholder={
overlay ? "Type your edits for your image" : "Send a message..."
}
className="aui-composer-input composer-input"
minRows={1}
maxRows={12}
autoFocus={!disabled}
disabled={disabled}
aria-label="Message input"
aria-label={overlay ? "Image edit instructions" : "Message input"}
// dir="auto": browser picks LTR/RTL from the first strong char;
// no effect on Latin / CJK / Devanagari.
dir="auto"
Expand Down Expand Up @@ -1107,7 +1109,7 @@ const ImagesToggle: FC = () => {
: "Enable image generation"
}
>
<ImageIcon className="size-3.5" />
<HugeiconsIcon icon={Image03Icon} className="size-3.5" strokeWidth={2} />
<span>Images</span>
</button>
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { cn } from "@/lib/utils";
import type { ToolCallMessagePartComponent } from "@assistant-ui/react";
import { DownloadIcon, ImageIcon, PencilIcon } from "lucide-react";
import type { CSSProperties, MouseEvent } from "react";
import { memo, useState } from "react";
import { memo, useCallback, useEffect, useRef, useState } from "react";
import { useGeneratedImageOverlay } from "./generated-image-overlay-context";
import { Image, downloadImagePart } from "./image";
import {
Expand Down Expand Up @@ -63,6 +63,8 @@ type GeneratedImagePart = {
filename?: string;
};

const CAPTION_COLLAPSED_LINES = 4;

const extensionForMime = (mime: string): string => {
switch (mime.toLowerCase()) {
case "image/jpeg":
Expand Down Expand Up @@ -119,7 +121,7 @@ function GeneratedImagePlaceholder({ label }: { label: string }) {
return (
<div
className={cn(
"generated-image-loading-card flex aspect-square w-[480px] max-w-full items-center justify-center rounded-2xl border border-border/70 bg-muted/15",
"generated-image-loading-card flex aspect-square w-[480px] max-w-full items-center justify-center rounded-2xl bg-muted/20 shadow-[0_0_12px_rgba(15,23,42,0.05),0_6px_18px_rgba(15,23,42,0.04)] dark:shadow-[0_0_12px_rgba(0,0,0,0.18),0_6px_18px_rgba(0,0,0,0.12)]",
)}
aria-busy="true"
aria-label={label}
Expand Down Expand Up @@ -154,6 +156,8 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
: null;
const imageTitle =
imageResult?.prompt?.trim() || prompt.trim() || "Generated image";
const captionPrompt = imageResult?.prompt?.trim() || prompt.trim();
const promptLikelyNeedsExpansion = captionPrompt.length > 220;
const imageMetadata = [imageResult?.size, imageResult?.quality, mime]
.filter(Boolean)
.join(" · ");
Expand All @@ -174,8 +178,63 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
: null;

const [open, setOpen] = useState(true);
const [expandedCaptionPrompt, setExpandedCaptionPrompt] = useState<
string | null
>(null);
const [promptOverflow, setPromptOverflow] = useState<{
prompt: string;
canExpand: boolean;
} | null>(null);
const captionRef = useRef<HTMLDivElement | null>(null);
const isPendingImage = !imagePart && status?.type === "running";

const promptOverflowMeasured = promptOverflow?.prompt === captionPrompt;
const promptCanExpand = promptOverflowMeasured
? promptOverflow.canExpand
: false;
const promptExpanded = expandedCaptionPrompt === captionPrompt;

const updatePromptOverflow = useCallback(() => {
const captionElement = captionRef.current;
if (!captionElement || !captionPrompt) {
return;
}
const computedStyle = window.getComputedStyle(captionElement);
const lineHeight = Number.parseFloat(computedStyle.lineHeight);
const collapsedHeight =
(Number.isFinite(lineHeight) ? lineHeight : 20) *
CAPTION_COLLAPSED_LINES;
const hasOverflow = captionElement.scrollHeight > collapsedHeight + 1;
setPromptOverflow((current) =>
current?.prompt === captionPrompt && current.canExpand === hasOverflow
? current
: { prompt: captionPrompt, canExpand: hasOverflow },
);
}, [captionPrompt]);

useEffect(() => {
const captionElement = captionRef.current;
if (!captionElement || !captionPrompt) {
return;
}
const frame = window.requestAnimationFrame(updatePromptOverflow);
const resizeObserver =
typeof ResizeObserver === "undefined"
? null
: new ResizeObserver(updatePromptOverflow);
resizeObserver?.observe(captionElement);
window.addEventListener("resize", updatePromptOverflow);
return () => {
window.cancelAnimationFrame(frame);
resizeObserver?.disconnect();
window.removeEventListener("resize", updatePromptOverflow);
};
}, [captionPrompt, updatePromptOverflow]);

const shouldClampPrompt =
(promptOverflowMeasured ? promptCanExpand : promptLikelyNeedsExpansion) &&
!promptExpanded;

const runningLabel = "Generating image…";
const completedLabel = formatGeneratedImageLabel(prompt);

Expand Down Expand Up @@ -231,21 +290,28 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
<ToolFallbackContent>
{imagePart ? (
<figure className="m-0 flex flex-col gap-2">
<div className="group/generated-image relative aspect-square w-[480px] max-w-full overflow-hidden rounded-2xl border border-border/70 bg-muted/30 shadow-sm">
<div className="group/generated-image relative aspect-square w-[480px] max-w-full overflow-hidden rounded-2xl bg-muted/25 shadow-lg shadow-foreground/5 dark:shadow-black/25">
<img
src={imagePart.image}
alt=""
aria-hidden={true}
className="pointer-events-none absolute inset-0 size-full scale-110 object-cover opacity-25 blur-2xl saturate-125"
Comment thread
wasimysaid marked this conversation as resolved.
/>
<div className="pointer-events-none absolute inset-0 bg-background/45" />
<button
type="button"
className="block size-full cursor-zoom-in overflow-hidden rounded-2xl focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
className="relative z-10 block size-full cursor-zoom-in overflow-hidden rounded-2xl focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
onClick={showPreview}
aria-label="Open generated image preview"
>
<Image.Preview
src={imagePart.image}
alt={imageTitle}
containerClassName="size-full min-h-0 bg-background"
className="size-full object-cover"
containerClassName="flex size-full min-h-0 items-center justify-center bg-transparent"
className="size-full object-contain"
/>
</button>
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex items-end justify-between gap-2 bg-gradient-to-t from-black/55 via-black/20 to-transparent p-3 opacity-100 transition-opacity sm:opacity-0 sm:group-hover/generated-image:opacity-100 sm:group-focus-within/generated-image:opacity-100">
<div className="pointer-events-none absolute inset-x-0 bottom-0 z-20 flex items-end justify-between gap-2 bg-gradient-to-t from-black/55 via-black/20 to-transparent p-3 opacity-100 transition-opacity sm:opacity-0 sm:group-hover/generated-image:opacity-100 sm:group-focus-within/generated-image:opacity-100">
<Button
type="button"
variant="dark"
Expand All @@ -268,9 +334,31 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
</Button>
</div>
</div>
{prompt ? (
<figcaption className="max-w-[480px] text-xs leading-snug text-muted-foreground">
{prompt}
{captionPrompt ? (
<figcaption className="max-w-[480px] text-xs leading-5 text-muted-foreground">
<div
ref={captionRef}
className={cn(
"whitespace-pre-wrap break-words",
shouldClampPrompt && "max-h-20 overflow-hidden",
)}
>
{captionPrompt}
</div>
{promptCanExpand ? (
<button
type="button"
className="mt-2 inline-flex text-xs font-medium text-foreground/80 underline-offset-4 hover:text-foreground hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
onClick={() =>
setExpandedCaptionPrompt((value) =>
value === captionPrompt ? null : captionPrompt,
)
}
aria-expanded={promptExpanded}
>
{promptExpanded ? "Show less" : "Show more"}
</button>
) : null}
</figcaption>
) : null}
</figure>
Expand Down
21 changes: 16 additions & 5 deletions studio/frontend/src/features/chat/api/chat-adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1433,6 +1433,17 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
// Tool call content parts — accumulated and yielded cumulatively.
// result is set directly on the tool-call part when tool_end arrives.
const toolCallParts: ToolCallMessagePart[] = [];
const orderAssistantContent = (
textParts: ReturnType<typeof parseAssistantContent>,
) => {
const imageToolParts = toolCallParts.filter(
(part) => part.toolName === "image_generation",
);
const otherToolParts = toolCallParts.filter(
(part) => part.toolName !== "image_generation",
);
return [...otherToolParts, ...textParts, ...imageToolParts];
};
// Anthropic document_citations tool_event payload, converted to
// Sources-panel source parts at end-of-stream so the inline [N]
// markers have matching entries.
Expand Down Expand Up @@ -2036,10 +2047,11 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
};
}
}
// Yield cumulative state so tool UI updates (tools first, text after)
// Yield cumulative state so tool UI updates. Search/code tools stay
// before the text, while generated images sit after the answer.
const textParts = parseAssistantContent(cumulativeText);
yield {
content: [...toolCallParts, ...textParts],
content: orderAssistantContent(textParts),
metadata: {
timing: buildTiming(
streamStartTime,
Expand Down Expand Up @@ -2187,7 +2199,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {

if (parts.length > 0 || toolCallParts.length > 0) {
yield {
content: [...toolCallParts, ...parts],
content: orderAssistantContent(parts),
metadata: {
timing: buildTiming(
streamStartTime,
Expand Down Expand Up @@ -2283,8 +2295,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {

yield {
content: [
...toolCallParts,
...parseAssistantContent(cumulativeText),
...orderAssistantContent(parseAssistantContent(cumulativeText)),
...sourceParts,
...documentCitationParts,
],
Expand Down
21 changes: 19 additions & 2 deletions studio/frontend/src/features/chat/shared-composer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,20 @@ import { isTauri } from "@/lib/api-base";
import { isMultimodalResponse } from "./types/api";
import { getImageInputUnavailableReason } from "./utils/image-input-support";
import { useAui } from "@assistant-ui/react";
import { ArrowUpIcon, DownloadIcon, GlobeIcon, HeadphonesIcon, ImageIcon, LightbulbIcon, LightbulbOffIcon, MicIcon, PlusIcon, SquareIcon, XIcon } from "lucide-react";
import {
ArrowUpIcon,
DownloadIcon,
GlobeIcon,
HeadphonesIcon,
LightbulbIcon,
LightbulbOffIcon,
MicIcon,
PlusIcon,
SquareIcon,
XIcon,
} from "lucide-react";
import { Image03Icon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { toast } from "@/lib/toast";
import { loadModel, validateModel } from "./api/chat-api";
import { parseExternalModelId, providerTypeSupportsVision } from "./external-providers";
Expand Down Expand Up @@ -1115,7 +1128,11 @@ export function SharedComposer({
imageToolsEnabled ? "Disable image generation" : "Enable image generation"
}
>
<ImageIcon className="size-3.5" />
<HugeiconsIcon
icon={Image03Icon}
className="size-3.5"
strokeWidth={2}
/>
<span>Images</span>
</button>
)}
Expand Down