Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions packages/client-runtime/src/work-log/itemDetail.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import { describe, expect, it } from "vite-plus/test";
import { ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";

import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";

function toolItem(output: unknown): OrchestrationV2TurnItem {
return {
id: TurnItemId.make("images"),
threadId: ThreadId.make("thread"),
runId: null,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 1,
status: "completed",
title: null,
startedAt: null,
completedAt: null,
updatedAt: DateTime.makeUnsafe("2026-10-01T00:00:00Z"),
type: "dynamic_tool",
toolName: "screenshots",
input: {},
output,
};
}

describe("tool image output parsing", () => {
it.each(["array", "content"] as const)(
"keeps markers for images beyond the served positions in %s output",
(shape) => {
// Detail responses retain image metadata after the server strips image bytes.
const image = { type: "image", mimeType: "image/png" };
const blocks = Array.from({ length: 8 }, () => ({ ...image }));
const wrap = (content: unknown[]) => (shape === "array" ? content : { content });
const withinLimit = toolItem(wrap(blocks));
expect(turnItemOutputText(withinLimit)).toBeNull();
expect(turnItemOutputImages(withinLimit)).toHaveLength(8);

const overflow = toolItem(
wrap([
...blocks,
{ type: "text", text: "Two additional screenshots:" },
{ ...image, sizeBytes: 100 },
{ type: "image", source: { type: "base64", media_type: "image/jpeg" } },
]),
);
expect(turnItemOutputImages(overflow)).toHaveLength(8);
expect(turnItemOutputText(overflow)).toBe("Two additional screenshots:\n[image]\n[image]");
},
);
});
19 changes: 12 additions & 7 deletions packages/client-runtime/src/work-log/itemDetail.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { AssetResource, OrchestrationV2TurnItem } from "@t3tools/contracts";
import { readToolOutputImage, toolOutputImages } from "@t3tools/shared/toolOutput";
import { toolOutputImageBlocks, toolOutputImages } from "@t3tools/shared/toolOutput";
import * as DateTime from "effect/DateTime";

const MAX_TEXT_BLOCK_DEPTH = 4;
Expand All @@ -12,28 +12,33 @@ function isSummarizedValue(value: unknown): boolean {
return isRecord(value) && value.truncated === true && typeof value.summary === "string";
}

function textFromBlocks(value: unknown, depth: number): string | null {
function textFromBlocks(
value: unknown,
depth: number,
imageBlocks: ReadonlyArray<unknown>,
): string | null {
if (depth > MAX_TEXT_BLOCK_DEPTH) return null;
if (typeof value === "string") return value;
if (Array.isArray(value)) {
const parts = value.map((block) => textFromBlocks(block, depth + 1));
const parts = value.map((block) => textFromBlocks(block, depth + 1, imageBlocks));
return parts.every((part) => part !== null)
? parts.filter((part) => part !== "").join("\n")
: null;
}
if (!isRecord(value)) return null;
if (value.type === "text" && typeof value.text === "string") return value.text;
if (value.type === "image") return readToolOutputImage(value) ? "" : "[image]";
if (value.type === "image") return imageBlocks.includes(value) ? "" : "[image]";
if (value.type === "resource_link" && typeof value.uri === "string") return value.uri;
if (value.type === "resource" && isRecord(value.resource)) {
const resource = value.resource;
if (typeof resource.text === "string") return resource.text;
if (typeof resource.uri === "string") return resource.uri;
}
const keys = Object.keys(value).filter((key) => key !== "isError" && key !== "is_error");
if (keys.length === 1 && keys[0] === "content") return textFromBlocks(value.content, depth + 1);
if (keys.length === 1 && keys[0] === "content")
return textFromBlocks(value.content, depth + 1, imageBlocks);
if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {
const text = textFromBlocks(value.content, depth + 1);
const text = textFromBlocks(value.content, depth + 1, imageBlocks);
return text?.trim() ? text : null;
}
return null;
Expand Down Expand Up @@ -61,7 +66,7 @@ function prettyJsonText(text: string): string {
function formatToolValue(value: unknown, hasImagePreview = false): string | null {
if (value === undefined || value === null) return null;
if (hasImagePreview && isRecord(value) && value.type === "image") return null;
const text = textFromBlocks(value, 0);
const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
if (text !== null) return text.trim() ? prettyJsonText(text) : null;
let json: string | undefined;
try {
Expand Down
Loading