Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/transcript-live-attachments-goal-clear.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": patch
---

Project prompt attachments into the live transcript (turn.started now carries session-media references) and clear the transcript goal when the goal is cleared.
2 changes: 2 additions & 0 deletions packages/agent-core-v2/src/agent/loop/loopService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ import {
isDisplayablePromptOrigin,
ThinkingDelta,
ToolCallDelta,
turnPromptAttachments,
turnPromptText,
TurnStarted,
TurnStepCompleted,
Expand Down Expand Up @@ -454,6 +455,7 @@ export class AgentLoopService extends Disposable implements IAgentLoopService {
turnId: job.turn.id,
origin,
prompt: isDisplayablePromptOrigin(origin) ? turnPromptText(job.seed.input, origin) : undefined,
promptAttachments: turnPromptAttachments(job.seed.input),
}),
);
void this.runTurn(job.turn, job.ready).then(job.result.resolve, job.result.reject);
Expand Down
28 changes: 28 additions & 0 deletions packages/agent-core-v2/src/agent/loop/turnEvents.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/* oxlint-disable typescript-eslint/no-unsafe-declaration-merging, eslint-plugin-import/namespace -- Event2 class+payload-interface declaration merging is the sanctioned event-declaration idiom. */
import type { PromptOrigin } from '#/agent/contextMemory/types';
import { parseDaemonFileUrl } from '#/agent/media/mediaRef';
import { Event2 } from '#/app/event/event2';
import type { FinishReason } from '#/kosong/contract/provider';
import type { ContentPart, TextPart } from '#/kosong/contract/message';
Expand All @@ -19,6 +20,7 @@ export interface TurnStartedPayload {
readonly turnId: number;
readonly origin: PromptOrigin;
readonly prompt?: string;
readonly promptAttachments?: readonly { kind: 'image' | 'video' | 'audio'; fileId: string }[];
}

export class TurnStarted extends Event2<TurnStartedPayload> {
Expand All @@ -40,6 +42,32 @@ export function turnPromptText(
return text.length > 0 ? text : undefined;
}

/** Media parts become the turn's transcript attachments only when they point
* at a session upload — the id must match the part's daemon file URL (a
* provider-issued id on a remote URL is not a session-media file id). */
export function turnPromptAttachments(
input: readonly ContentPart[],
): TurnStartedPayload['promptAttachments'] {
const attachments: { kind: 'image' | 'video' | 'audio'; fileId: string }[] = [];
const sessionMediaFileId = (url: string, id: string | undefined): string | undefined => {
if (id === undefined) return undefined;
return parseDaemonFileUrl(url)?.fileId === id ? id : undefined;
};
for (const part of input) {
if (part.type === 'image_url') {
const fileId = sessionMediaFileId(part.imageUrl.url, part.imageUrl.id);
if (fileId !== undefined) attachments.push({ kind: 'image', fileId });
} else if (part.type === 'video_url') {
const fileId = sessionMediaFileId(part.videoUrl.url, part.videoUrl.id);
if (fileId !== undefined) attachments.push({ kind: 'video', fileId });
} else if (part.type === 'audio_url') {
const fileId = sessionMediaFileId(part.audioUrl.url, part.audioUrl.id);
if (fileId !== undefined) attachments.push({ kind: 'audio', fileId });
}
}
return attachments.length > 0 ? attachments : undefined;
}

export function isDisplayablePromptOrigin(origin: PromptOrigin): boolean {
if (origin.kind === 'user') return true;
return (
Expand Down
37 changes: 37 additions & 0 deletions packages/agent-core-v2/test/agent/loop/loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,43 @@ describe('Agent loop', () => {

expect(prompts).toEqual([undefined, 'hi']);
});

it('carries session-media prompt attachments on turn.started', async () => {
const payloads: Array<readonly { kind: string; fileId: string }[] | undefined> = [];
const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => {
payloads.push(event.promptAttachments);
});
ctx.mockNextResponse({ type: 'text', text: 'seen' });

const turn = (
await loop.enqueue(
new MessageStepRequest(
{
role: 'user',
content: [
{ type: 'image_url', imageUrl: { url: 'kimi-file://file_1', id: 'file_1' } },
{ type: 'video_url', videoUrl: { url: 'kimi-file://file_2', id: 'file_2' } },
{ type: 'image_url', imageUrl: { url: 'https://example.com/no-id.png' } },
{ type: 'image_url', imageUrl: { url: 'ms://provider-blob', id: 'prov_1' } },
{ type: 'text', text: 'look' },
],
toolCalls: [],
origin: { kind: 'user' },
},
{ admission: 'newTurn' },
),
).assigned
).turn;
await turn.result;
subscription.dispose();

expect(payloads).toEqual([
[
{ kind: 'image', fileId: 'file_1' },
{ kind: 'video', fileId: 'file_2' },
],
]);
});
});

describe('turn telemetry', () => {
Expand Down
4 changes: 2 additions & 2 deletions packages/kap-server/src/lib/promptMedia.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,10 @@ export function contentToCoreParts(content: WireContent): ContentPart[] {
if (part.type === 'text') parts.push({ type: 'text', text: part.text });
else if (part.type === 'image' && part.source.kind === 'url') parts.push({ type: 'image_url', imageUrl: { url: part.source.url, id: part.source.id } });
else if (part.type === 'image' && part.source.kind === 'base64') parts.push({ type: 'image_url', imageUrl: { url: `data:${part.source.media_type};base64,${part.source.data}` } });
else if (part.type === 'image' && part.source.kind === 'session_media') parts.push({ type: 'image_url', imageUrl: { url: buildDaemonFileUrl(part.source.file_id) } });
else if (part.type === 'image' && part.source.kind === 'session_media') parts.push({ type: 'image_url', imageUrl: { url: buildDaemonFileUrl(part.source.file_id), id: part.source.file_id } });
else if (part.type === 'video' && part.source.kind === 'url') parts.push({ type: 'video_url', videoUrl: { url: part.source.url, id: part.source.id } });
else if (part.type === 'video' && part.source.kind === 'base64') parts.push({ type: 'video_url', videoUrl: { url: `data:${part.source.media_type};base64,${part.source.data}` } });
else if (part.type === 'video' && part.source.kind === 'session_media') parts.push({ type: 'video_url', videoUrl: { url: buildDaemonFileUrl(part.source.file_id) } });
else if (part.type === 'video' && part.source.kind === 'session_media') parts.push({ type: 'video_url', videoUrl: { url: buildDaemonFileUrl(part.source.file_id), id: part.source.file_id } });
}
return parts;
}
Expand Down
3 changes: 3 additions & 0 deletions packages/kap-server/src/protocol/events-zod.ts
Original file line number Diff line number Diff line change
Expand Up @@ -696,6 +696,9 @@ export const turnStartedEventSchema = z.object({
origin: promptOriginSchema,
prompt: z.string().optional(),
promptId: z.string().optional(),
promptAttachments: z
.array(z.object({ kind: z.enum(['image', 'video', 'audio']), fileId: z.string() }))
.optional(),
});

export const turnEndedEventSchema = z.object({
Expand Down
4 changes: 3 additions & 1 deletion packages/kap-server/src/services/transcript/coreEventMap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1081,7 +1081,9 @@ export class AgentTranscriptProjector {
}): TranscriptOperation[] {
const ops: TranscriptOperation[] = [];
const snapshot = event.snapshot;
if (snapshot !== null) {
if (snapshot === null) {
ops.push({ op: 'meta.merge', meta: { goal: null } });
} else {
ops.push({
op: 'meta.merge',
meta: {
Expand Down
1 change: 1 addition & 0 deletions packages/kap-server/test/prompts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -730,6 +730,7 @@ describe('server-v2 /api/v1 prompts', () => {
type: 'image_url',
imageUrl: {
url: `kimi-file://${uploaded.id}`,
id: uploaded.id,
},
});
});
Expand Down
4 changes: 3 additions & 1 deletion packages/kap-server/test/services/transcript.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -997,7 +997,9 @@ describe('AgentTranscriptProjector', () => {
expect(marker).toMatchObject({ marker: 'goal', payload: { snapshot } });

const clearedOps = projector.map(ev({ type: 'goal.updated', snapshot: null }));
expect(clearedOps.every((op) => op.op === 'marker.upsert')).toBe(true);
expect(clearedOps[0]).toEqual({ op: 'meta.merge', meta: { goal: null } });
tx.apply(clearedOps);
expect(tx.getMeta().goal).toBeUndefined();
});

it('mirrors plan / swarm mode slices into meta.modes (only when provided)', () => {
Expand Down
5 changes: 5 additions & 0 deletions packages/protocol/src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,8 @@ export interface TurnStartedEvent {
readonly prompt?: string;
/** The prompt record id when the turn was opened by a prompt submission. */
readonly promptId?: string;
/** Session-media references carried by the prompt (transcript attachments). */
readonly promptAttachments?: readonly { kind: 'image' | 'video' | 'audio'; fileId: string }[];
}

export interface TurnEndedEvent {
Expand Down Expand Up @@ -1647,6 +1649,9 @@ export const turnStartedEventSchema = z.object({
origin: promptOriginSchema,
prompt: z.string().optional(),
promptId: z.string().optional(),
promptAttachments: z
.array(z.object({ kind: z.enum(['image', 'video', 'audio']), fileId: z.string() }))
.optional(),
}) satisfies z.ZodType<TurnStartedEvent>;

export const turnEndedEventSchema = z.object({
Expand Down
2 changes: 2 additions & 0 deletions packages/transcript/src/contract/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,9 @@ export const transcriptMetaSchema = z.object({
agent: agentStatusMetaSchema.optional(),
});

/** `goal` set to `null` in a merge clears the goal (same convention as mode keys). */
export const transcriptMetaMergeSchema = transcriptMetaSchema.extend({
goal: goalMetaSchema.nullable().optional(),
modes: modesMetaMergeSchema.optional(),
});

Expand Down
5 changes: 3 additions & 2 deletions packages/transcript/src/model/meta.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ export interface TranscriptMeta {
readonly agent?: AgentStatusMeta;
}

/** Contract shape of a `meta.merge` payload — like {@link TranscriptMeta}, but mode keys may be `null` to clear. */
export type TranscriptMetaMerge = Omit<TranscriptMeta, 'modes'> & {
/** Contract shape of a `meta.merge` payload — like {@link TranscriptMeta}, but mode keys and `goal` may be `null` to clear. */
export type TranscriptMetaMerge = Omit<TranscriptMeta, 'modes' | 'goal'> & {
readonly modes?: ModesMetaMerge;
readonly goal?: GoalMeta | null;
};
2 changes: 1 addition & 1 deletion packages/transcript/src/ops/apply.ts
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,7 @@ function applyMetaMerge(state: AgentState, meta: TranscriptMetaMerge): ApplyResu
const agent =
meta.agent !== undefined ? { ...state.meta.agent, ...meta.agent } : state.meta.agent;
const next: TranscriptMeta = {
goal: meta.goal ?? state.meta.goal,
goal: meta.goal === null ? undefined : (meta.goal ?? state.meta.goal),
activity: meta.activity ?? state.meta.activity,
modes: modes !== undefined && modes.plan === undefined && modes.swarm === undefined ? undefined : modes,
agent,
Expand Down
Loading