Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/agent-core-v2/docs/state-manifest.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1052,6 +1052,7 @@ export interface AgentStateSnapshot {
type: 'think';
think: string;
encrypted?: string;
detailsIndex?: number;
} | /* ImageURLPart — packages/agent-core-v2/src/human/llm/message.ts */ {
type: 'image_url';
imageUrl: {
Expand Down Expand Up @@ -1341,6 +1342,7 @@ export interface AgentStateSnapshot {
type: 'think';
think: string;
encrypted?: string;
detailsIndex?: number;
} | /* ImageURLPart — packages/agent-core-v2/src/human/llm/message.ts */ {
type: 'image_url';
imageUrl: {
Expand Down
18 changes: 16 additions & 2 deletions packages/agent-core-v2/src/agent/loop/loopService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -809,7 +809,12 @@ export class AgentLoopService extends Disposable implements IAgentLoopService {
);
return;
case 'thinking':
this.accumulateMachinePart(turn, { type: 'think', think: delta.delta });
this.accumulateMachinePart(turn, {
type: 'think',
think: delta.delta,
encrypted: delta.encrypted,
detailsIndex: delta.detailsIndex,
});
void this.dispatcher.dispatch(
new ThinkingDelta({ agentId: this.scopeContext.agentId, turnId: turn.id, delta: delta.delta }),
);
Expand Down Expand Up @@ -1085,7 +1090,16 @@ export class AgentLoopService extends Disposable implements IAgentLoopService {
}

private drainMachinePartials(turn: ActiveTurn, step: MachineStepState): void {
for (const part of turn.partials.splice(0).filter((entry) => !isVacuousContentPart(entry))) {
const drained = turn.partials.splice(0).filter((entry) => !isVacuousContentPart(entry));
let lastCompleteThink = -1;
for (const [index, part] of drained.entries()) {
if (part.type === 'think' && part.encrypted !== undefined) {
lastCompleteThink = index;
}
}
for (const part of drained.filter(
(part, index) => part.type !== 'think' || index <= lastCompleteThink,
)) {
this.context.appendLoopEvent({
type: 'content.part',
uuid: randomUUID(),
Expand Down
14 changes: 12 additions & 2 deletions packages/agent-core-v2/src/agent/loop/machine/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@ import { createMachineTools, type ToolResultExtras } from './tools';

export type MachineEngineDelta =
| { readonly kind: 'assistant'; readonly delta: string }
| { readonly kind: 'thinking'; readonly delta: string }
| {
readonly kind: 'thinking';
readonly delta: string;
readonly encrypted?: string;
readonly detailsIndex?: number;
}
| {
readonly kind: 'toolCall';
readonly toolCallId: string;
Expand Down Expand Up @@ -151,7 +156,12 @@ function createDeltaSplitter(): (part: StreamedMessagePart) => MachineEngineDelt
case 'text':
return { kind: 'assistant', delta: part.text };
case 'think':
return { kind: 'thinking', delta: part.think };
return {
kind: 'thinking',
delta: part.think,
encrypted: part.encrypted,
detailsIndex: part.detailsIndex,
};
case 'image_url':
case 'audio_url':
case 'video_url':
Expand Down
4 changes: 4 additions & 0 deletions packages/agent-core-v2/src/human/llm/message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export interface ThinkPart {
type: 'think';
think: string;
encrypted?: string;
detailsIndex?: number;
}

export interface ImageURLPart {
Expand Down Expand Up @@ -103,6 +104,9 @@ export function mergeInPlace(target: StreamedMessagePart, source: StreamedMessag
if (target.encrypted !== undefined) {
return false;
}
if (target.detailsIndex !== source.detailsIndex) {
return false;
}
target.think += source.think;
if (source.encrypted !== undefined) {
target.encrypted = source.encrypted;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,12 @@ import type { TokenUsage } from '#/llm/usage';

import { lowerMessage, type OpenAIWireMessage } from './lower';
import { extractToolMedia } from './patterns';
import { DEFAULT_REASONING_KEY, extractReasoning } from './reasoning-key';
import {
convertReasoningDetails,
DEFAULT_REASONING_KEY,
extractReasoning,
extractReasoningDetails,
} from './reasoning-key';

function responseFormatToOpenAI(format: ResponseFormat): Record<string, unknown> {
if (format.type === 'json_object') {
Expand Down Expand Up @@ -265,6 +270,7 @@ export const openAIFormat: ProtocolFormat<OpenAIRequestParams, RawResponse, RawC
},

createStreamParser(options?: StreamParserOptions) {
const explicitReasoningKey = options?.trait?.reasoningKey?.(options.ctx);
const bufferedToolCalls = new Map<number | string, BufferedStreamToolCall>();

function convertStreamToolCall(toolCall: RawStreamToolCallDelta): StreamedMessagePart[] {
Expand Down Expand Up @@ -351,9 +357,17 @@ export const openAIFormat: ProtocolFormat<OpenAIRequestParams, RawResponse, RawC
if (!delta) {
return;
}
const reasoning = extractReasoning(delta);
if (reasoning !== undefined) {
sink.onDelta({ type: 'think', think: reasoning.value });
const reasoningDetails =
explicitReasoningKey === undefined ? extractReasoningDetails(delta) : undefined;
if (reasoningDetails !== undefined) {
for (const part of convertReasoningDetails(reasoningDetails)) {
sink.onDelta(part);
}
} else {
const reasoning = extractReasoning(delta);
if (reasoning !== undefined) {
sink.onDelta({ type: 'think', think: reasoning.value });
}
}
if (typeof delta.content === 'string' && delta.content.length > 0) {
sink.onDelta({ type: 'text', text: delta.content });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { extractText, type ContentPart, type Message } from '#/llm/message';
import type { ProtocolTrait, TraitContext } from '#/llm/protocol/trait';

import { TOOL_RESULT_MEDIA_PLACEHOLDER } from './patterns';
import { DEFAULT_REASONING_KEY, REASONING_DETAILS_KEY } from './reasoning-key';

export type OpenAIContentPart = {
type: 'text' | 'image_url' | 'audio_url' | 'video_url';
Expand Down Expand Up @@ -140,7 +141,20 @@ export function lowerMessage(message: Message, lower: OpenAILowerContext): OpenA
} else {
converted = { role: message.role, content: content ?? '' };
}
if (hasReasoningPart || (preserveThinking && message.role === 'assistant')) {
const reasoningDetails: Record<string, unknown>[] = [];
for (const part of message.content) {
if (part.type !== 'think' || part.detailsIndex === undefined) continue;
if (part.think.length > 0) {
reasoningDetails.push({ type: 'summary', summary: part.think });
}
if (part.encrypted !== undefined) {
reasoningDetails.push({ type: 'encrypted', encrypted: part.encrypted });
}
}
if (reasoningDetails.length > 0) {
(converted as Record<string, unknown>)[REASONING_DETAILS_KEY] = reasoningDetails;
(converted as Record<string, unknown>)[DEFAULT_REASONING_KEY] = reasoningContent;
} else if (hasReasoningPart || (preserveThinking && message.role === 'assistant')) {
(converted as Record<string, unknown>)[reasoningKey] = reasoningContent;
}
const hooked =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import type { StreamedMessagePart, ThinkPart } from '#/llm/message';

export const KNOWN_REASONING_KEYS = [
'reasoning_content',
'reasoning_details',
Expand Down Expand Up @@ -40,3 +42,60 @@ export class ReasoningKeyDialect {
return this._explicitKey ?? this._detected ?? DEFAULT_REASONING_KEY;
}
}

export const REASONING_DETAILS_KEY = 'reasoning_details';

export interface ReasoningDetailsElement {
readonly type?: string;
readonly index: number;
readonly summary?: string;
readonly encrypted?: string;
}

function toReasoningDetailsElement(
value: unknown,
position: number,
): ReasoningDetailsElement | undefined {
if (typeof value !== 'object' || value === null) return undefined;
const record = value as Record<string, unknown>;
const type = typeof record['type'] === 'string' ? record['type'] : undefined;
if (type !== undefined && type !== 'summary' && type !== 'encrypted') return undefined;
const index = typeof record['index'] === 'number' ? record['index'] : position;
const summary = typeof record['summary'] === 'string' ? record['summary'] : undefined;
const encrypted = typeof record['encrypted'] === 'string' ? record['encrypted'] : undefined;
return { type, index, summary, encrypted };
}

export function extractReasoningDetails(
source: unknown,
): ReasoningDetailsElement[] | undefined {
if (typeof source !== 'object' || source === null) return undefined;
const value = (source as Record<string, unknown>)[REASONING_DETAILS_KEY];
if (!Array.isArray(value)) return undefined;
const elements: ReasoningDetailsElement[] = [];
for (const [position, item] of value.entries()) {
const element = toReasoningDetailsElement(item, position);
if (element !== undefined) elements.push(element);
}
return elements;
}

export function convertReasoningDetails(
elements: readonly ReasoningDetailsElement[],
): StreamedMessagePart[] {
const parts: StreamedMessagePart[] = [];
for (const element of elements) {
if (element.type !== 'encrypted' && element.summary !== undefined && element.summary.length > 0) {
parts.push({ type: 'think', think: element.summary, detailsIndex: element.index } satisfies ThinkPart);
}
if (element.type !== 'summary' && element.encrypted !== undefined && element.encrypted.length > 0) {
parts.push({
type: 'think',
think: '',
encrypted: element.encrypted,
detailsIndex: element.index,
} satisfies ThinkPart);
}
}
return parts;
}
96 changes: 95 additions & 1 deletion packages/agent-core-v2/src/human/test/llm/thinking.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ describe('openai requester thinking', () => {
expect(client.body()['reasoning_effort']).toBe('medium');
});

it('echoes think parts under reasoning_content by default', async () => {
it('echoes think parts under reasoning_content by default and restores marked reasoning_details', async () => {
const client = stubOpenAIClient(chatCompletionChunks());
const requester = createOpenAIRequester(undefined, { clientFactory: client.clientFactory });
await requester.generate(
Expand All @@ -351,6 +351,33 @@ describe('openai requester thinking', () => {
const assistant = bodyMessages(client.body())[1]!;
expect(assistant['reasoning_content']).toBe('abc');
expect(assistant['content']).toBe('hello');

const marked = stubOpenAIClient(chatCompletionChunks());
const markedRequester = createOpenAIRequester(undefined, {
clientFactory: marked.clientFactory,
});
await markedRequester.generate(
{ model, thinking: { effort: 'off' } },
{
messages: [
createUserMessage('hi'),
createAssistantMessage([
{ type: 'think', think: '第一段续', detailsIndex: 0 },
{ type: 'think', think: '第二段', detailsIndex: 1 },
{ type: 'think', think: '', encrypted: 'cipher', detailsIndex: 2 },
{ type: 'text', text: 'ok' },
]),
],
},
{ signal: new AbortController().signal },
);
const markedAssistant = bodyMessages(marked.body())[1]!;
expect(markedAssistant['reasoning_details']).toEqual([
{ type: 'summary', summary: '第一段续' },
{ type: 'summary', summary: '第二段' },
{ type: 'encrypted', encrypted: 'cipher' },
]);
expect(markedAssistant['reasoning_content']).toBe('第一段续第二段');
});

it('echoes an empty reasoning_content on think-less assistant messages only when keeping all', async () => {
Expand Down Expand Up @@ -426,6 +453,34 @@ describe('openai requester thinking', () => {
const detectedAssistant = bodyMessages(captured[1]!)[1]!;
expect(detectedAssistant['reasoning']).toBe('abc');
expect('reasoning_content' in detectedAssistant).toBe(false);

const explicit = stubOpenAIClient(
chatCompletionChunks([
{
reasoning_details: [
{ index: 0, type: 'summary', summary: 'ignored' },
{ index: 1, type: 'encrypted', encrypted: 'cipher' },
],
},
{ content: 'ok' },
]),
);
const explicitRequester = createOpenAIRequester(
{ reasoningKey: () => 'reasoning' },
{ clientFactory: explicit.clientFactory },
);
const explicitParts: unknown[] = [];
await explicitRequester.generate(
{ model },
{ messages },
{
signal: new AbortController().signal,
onEvent: (event) => {
if (event.type === 'llm.delta') explicitParts.push(event.part);
},
},
);
expect(explicitParts).toEqual([{ type: 'text', text: 'ok' }]);
});

it('parses reasoning from stream deltas', async () => {
Expand Down Expand Up @@ -470,5 +525,44 @@ describe('openai requester thinking', () => {
{ type: 'text', text: 'hi' },
{ type: 'think', think: '' },
]);
await expect(
collect(
chatCompletionChunks([
{
reasoning_content: '第一段',
reasoning_details: [{ index: 0, type: 'summary', summary: '第一段' }],
},
{
reasoning_details: [
{ index: 0, summary: '续' },
{ index: 1, type: 'summary', summary: '第二段' },
],
},
{ reasoning_details: [{ index: 2, type: 'encrypted', encrypted: 'cipher' }] },
{ content: 'ok' },
]),
),
).resolves.toEqual([
{ type: 'think', think: '第一段续', detailsIndex: 0 },
{ type: 'think', think: '第二段', detailsIndex: 1 },
{ type: 'think', think: '', encrypted: 'cipher', detailsIndex: 2 },
{ type: 'text', text: 'ok' },
]);
await expect(
collect(
chatCompletionChunks([
{
reasoning_details: [
{ index: 0, type: 'reasoning.text', text: 'foreign', format: 'unknown' },
{ index: 1, type: 'summary', summary: 'kept' },
],
},
{ content: 'ok' },
]),
),
).resolves.toEqual([
{ type: 'think', think: 'kept', detailsIndex: 1 },
{ type: 'text', text: 'ok' },
]);
});
});
Loading
Loading