feat(agent-core-v2): accept HEIC, HEIF, and BMP images for Kimi models

.changeset/kimi-heic-bmp-images.md

@@ -0,0 +1,5 @@
+---
+"@moonshot-ai/kimi-code": patch
+---
+
+Accept HEIC, HEIF, and BMP images in ReadMediaFile and prompt attachments when the model is served by Kimi.

apps/kimi-code/src/utils/clipboard/clipboard-image.ts

@@ -1,9 +1,10 @@
 /**
  * Read media from the system clipboard with graceful platform fallbacks.
  *
- * kimi-core's LLM pipeline only accepts PNG/JPEG/GIF/WebP, and the
- * clipboard sources we query already emit those formats on supported
- * platforms — so we deliberately do not include a BMP→PNG converter.
+ * Every model provider accepts PNG/JPEG/GIF/WebP (the engine widens the set
+ * per provider, e.g. BMP/HEIC/HEIF for Kimi), and the clipboard sources we
+ * query already emit those baseline formats on supported platforms — so we
+ * deliberately do not include a BMP→PNG converter.
  *
  * Lookup order:
  *   macOS file clipboard       -> osascript/AppKit file URLs

packages/acp-server/src/convert.ts

@@ -3,7 +3,6 @@ import {
   buildImageCompressionCaption,
   compressBase64ForModel,
   type ContentPart,
-  gateImageFormatParts,
   type McpServerConfig,
   parseImageDataUrl,
   persistOriginalImage,
@@ -84,18 +83,19 @@ export function acpBlocksToContentParts(blocks: readonly ContentBlock[]): readon
  * (`resolvePromptMediaFiles`). Best effort: a part that cannot be compressed
  * is passed through unchanged.
  *
- * This is NOT duplicated by the engine: agent-core-v2's prompt pipeline
+ * Compression is NOT duplicated by the engine: agent-core-v2's prompt pipeline
  * (`agent/prompt/promptService.ts`) only *extracts* pre-existing compression
  * captions from user text (rerouting them to system reminders) — it never
- * gates or compresses images at the prompt entry, so the edge ingestion point
- * owns the step.
+ * compresses images at the prompt entry, so the edge ingestion point owns
+ * that step.
  *
- * The format gate (`gateImageFormatParts`) runs first: parts whose MIME is
- * outside the provider-accepted set are never forwarded — the part is
- * dropped and a text notice stands in, so one unsupported image cannot
- * poison the session history; accepted MIME aliases (`image/jpg`,
- * case/whitespace variants) are rewritten to the canonical form strict
- * provider whitelists require.
+ * Format gating is deliberately left to the engine: the accepted image
+ * formats depend on the provider the agent is bound to, which this edge does
+ * not know. The engine's prompt pipeline gates every image part against that
+ * provider's set (dropping rejected parts for a text notice and rewriting
+ * accepted MIME aliases to their canonical form) before anything reaches the
+ * session history, so parts in formats we cannot re-encode pass through here
+ * untouched.
  *
  * Compression is never silent: a re-encoded image gains a caption text part
  * immediately before it stating what the original was, and the original bytes
@@ -118,7 +118,7 @@ export async function compressPromptImageParts(
   } = {},
 ): Promise<ContentPart[]> {
   const out: ContentPart[] = [];
-  for (const part of gateImageFormatParts(parts)) {
+  for (const part of parts) {
     if (part.type === 'image_url') {
       const parsed = parseImageDataUrl(part.imageUrl.url);
       if (parsed !== null) {

packages/acp-server/test/convert.test.ts

@@ -107,27 +107,18 @@ describe('compressPromptImageParts', () => {
     return { type: 'image_url', imageUrl: { url } };
   }
 
-  it('drops an unsupported image format and stands a text notice in', async () => {
-    // Zero bytes carry no magic number, so the declared MIME wins the
-    // effective-MIME resolution and the gate rejects it.
+  it('leaves format judgment to the engine: a format it cannot re-encode passes through', async () => {
+    // Which formats are acceptable depends on the provider the agent is
+    // bound to, which this edge does not know; the engine's prompt gate
+    // decides. So neither a HEIC payload nor a MIME alias is rewritten here.
     const heic = `data:image/heic;base64,${Buffer.alloc(32).toString('base64')}`;
-    const out = await compressPromptImageParts([{ type: 'text', text: 'look' }, imagePart(heic)]);
-    expect(out).toHaveLength(2);
-    expect(out[0]).toEqual({ type: 'text', text: 'look' });
-    const notice = out[1];
-    expect(notice?.type).toBe('text');
-    expect((notice as { text: string }).text).toContain('unsupported image format image/heic');
-  });
-
-  it('rewrites accepted MIME aliases to the canonical form', async () => {
-    const base64 = solidPngBase64(8, 8);
-    const out = await compressPromptImageParts([imagePart(`data:IMAGE/PNG;base64,${base64}`)]);
-    expect(out).toHaveLength(1);
-    const part = out[0];
-    expect(part?.type).toBe('image_url');
-    expect((part as { imageUrl: { url: string } }).imageUrl.url).toBe(
-      `data:image/png;base64,${base64}`,
-    );
+    const alias = `data:IMAGE/PNG;base64,${solidPngBase64(8, 8)}`;
+    const out = await compressPromptImageParts([
+      { type: 'text', text: 'look' },
+      imagePart(heic),
+      imagePart(alias),
+    ]);
+    expect(out).toEqual([{ type: 'text', text: 'look' }, imagePart(heic), imagePart(alias)]);
   });
 
   it('passes an under-limit image through unchanged and persists nothing', async () => {

packages/agent-core-v2/scripts/check-import-boundaries.mjs

@@ -23,6 +23,7 @@ const HUMAN_VOCABULARY = new Set([
   'llm/finish-reason',
   'llm/response-format',
   'llm/media/upload',
+  'llm/media/image-formats',
   'llm/requester/requester',
   'llm/toolCallIdNormalizer',
   'llm-kimi/trait',

packages/agent-core-v2/src/agent/mcp/mcpService.ts

@@ -8,6 +8,7 @@ import { type IDisposable } from "#/_base/di/lifecycle";
 import { Service } from "#/_base/di/service";
 import { ErrorCodes, makeErrorPayload } from "#/errors";
 import { abortable } from '#/_base/utils/abort';
+import { IAgentProfileService } from '#/agent/profile/profile';
 import { IAgentStateService } from '#/agent/state/agentState';
 import { ITelemetryService } from '#/app/telemetry/telemetry';
 import { sessionMediaOriginalsDir } from '#/agent/media/image-originals';
@@ -60,6 +61,7 @@ export class AgentMcpService extends Service implements IAgentMcpService {
     @ITelemetryService private readonly telemetry: ITelemetryService,
     @IAgentScopeContext private readonly scopeContext: IAgentScopeContext,
     @IAgentStateService private readonly states: IAgentStateService,
+    @IAgentProfileService private readonly profile: IAgentProfileService,
   ) {
     super();
     this.states.contributeState(mcpDiscoveryKey);
@@ -286,6 +288,7 @@ export class AgentMcpService extends Service implements IAgentMcpService {
           createMcpTool(qualified, tool, client, {
             originalsDir: sessionMediaOriginalsDir(this.sessionContext.sessionDir),
             telemetry: this.telemetry,
+            providerType: () => this.profile.getModelProviderType(),
             reconnect: (signal) => this.reconnectForToolCall(serverName, client, signal),
             isRemoved: () =>
               this.mcpHandle.connectionManager.get(serverName)?.status === 'removed',

packages/agent-core-v2/src/agent/mcp/output.ts

@@ -13,6 +13,7 @@ import type { MCPContentBlock, MCPToolResult } from '#/mcpCore/types';
 export interface McpOutputOptions {
   readonly originalsDir?: string;
   readonly telemetry?: ITelemetryService;
+  readonly providerType?: string;
 }
 
 export const MCP_MAX_BINARY_PART_BYTES = 10 * 1024 * 1024;
@@ -28,7 +29,7 @@ function droppedBlockNotice(reason: string): ContentPart {
   return { type: 'text', text: `[MCP content dropped: ${reason}]` };
 }
 
-export function convertMCPContentBlock(block: MCPContentBlock): ContentPart {
+export function convertMCPContentBlock(block: MCPContentBlock, providerType?: string): ContentPart {
   if (block.type === 'text' && typeof block.text === 'string') {
     return { type: 'text', text: block.text };
   }
@@ -85,8 +86,11 @@ export function convertMCPContentBlock(block: MCPContentBlock): ContentPart {
   if (block.type === 'resource_link' && typeof block.uri === 'string') {
     const mimeType = block.mimeType ?? 'application/octet-stream';
     if (mimeType.startsWith('image/')) {
-      if (!isModelAcceptedImageMime(mimeType)) {
-        return { type: 'text', text: buildUnsupportedImageNotice(mimeType, block.uri) };
+      if (!isModelAcceptedImageMime(mimeType, providerType)) {
+        return {
+          type: 'text',
+          text: buildUnsupportedImageNotice(mimeType, block.uri, providerType),
+        };
       }
       return { type: 'image_url', imageUrl: { url: block.uri } };
     }
@@ -111,7 +115,7 @@ export async function mcpResultToExecutableOutput(
 ): Promise<ExecutableToolResult> {
   const converted: ContentPart[] = [];
   for (const block of result.content) {
-    converted.push(convertMCPContentBlock(block));
+    converted.push(convertMCPContentBlock(block, options.providerType));
   }
 
   const wrapped = wrapMediaOnly(converted, qualifiedToolName);
@@ -141,6 +145,7 @@ export async function mcpResultToExecutableOutput(
   const compressed = await compressImageContentParts(wrapped, {
     telemetry: options.telemetry,
     telemetrySource: 'mcp_tool_result',
+    providerType: options.providerType,
     annotate: {
       persistOriginal: (bytes, mimeType) =>
         persistOriginalImage(

packages/agent-core-v2/src/agent/mcp/tools/mcp.ts

@@ -16,6 +16,7 @@ import {
 interface McpToolOptions {
   readonly originalsDir?: string;
   readonly telemetry?: ITelemetryService;
+  readonly providerType?: () => string | undefined;
   readonly reconnect?: (signal?: AbortSignal) => Promise<MCPClient | undefined>;
   readonly isRemoved?: () => boolean;
 }
@@ -52,6 +53,7 @@ export function createMcpTool(
         return mcpResultToExecutableOutput(result, qualifiedName, {
           originalsDir: options.originalsDir,
           telemetry: options.telemetry,
+          providerType: options.providerType?.(),
         });
       },
     }),

packages/agent-core-v2/src/agent/media/image-compress.ts

@@ -1,6 +1,7 @@
 import type { ContentPart } from '#human/llm/message';
 import type { ImageCompressEvent, ImageCropEvent } from '#/app/telemetry/events';
 import type { ITelemetryService } from '#/app/telemetry/telemetry';
+import { DEFAULT_INLINE_IMAGE_BYTE_BUDGET } from '#human/llm/media/image-formats';
 
 import { sniffImageDimensions } from './file-type';
 import {
@@ -28,7 +29,7 @@ export function resolveMaxImageEdgePx(): number {
   return configuredMaxImageEdgePx ?? MAX_IMAGE_EDGE_PX;
 }
 
-export const IMAGE_BYTE_BUDGET = 3.75 * 1024 * 1024;
+export const IMAGE_BYTE_BUDGET = DEFAULT_INLINE_IMAGE_BYTE_BUDGET;
 
 export const READ_IMAGE_BYTE_BUDGET = 256 * 1024;
 
@@ -58,6 +59,12 @@ export const MAX_IMAGE_DECODE_BYTES = 64 * 1024 * 1024;
 
 const RECODABLE_MIME = new Set(['image/png', 'image/jpeg', 'image/webp']);
 
+export function isRecodableImage(bytes: Uint8Array, mimeType: string): boolean {
+  const normalizedMime = normalizeImageMime(mimeType);
+  if (!RECODABLE_MIME.has(normalizedMime)) return false;
+  return normalizedMime !== 'image/webp' || !isAnimatedWebp(bytes);
+}
+
 export interface CompressImageOptions {
   readonly maxEdge?: number;
   readonly byteBudget?: number;
@@ -120,9 +127,7 @@ export async function compressImageForModel(
     return result;
   };
 
-  if (bytes.length === 0) return finish('passthrough_unsupported', passthrough());
-  if (!RECODABLE_MIME.has(normalizedMime)) return finish('passthrough_unsupported', passthrough());
-  if (normalizedMime === 'image/webp' && isAnimatedWebp(bytes)) {
+  if (bytes.length === 0 || !isRecodableImage(bytes, normalizedMime)) {
     return finish('passthrough_unsupported', passthrough());
   }
 
@@ -271,7 +276,10 @@ export interface CompressedContentParts {
   readonly captions: readonly string[];
 }
 
-export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] {
+export function gateImageFormatParts(
+  parts: readonly ContentPart[],
+  providerType?: string,
+): ContentPart[] {
   const out: ContentPart[] = [];
   for (const part of parts) {
     if (part.type === 'image_url') {
@@ -281,11 +289,11 @@ export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart
           out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) });
           continue;
         }
-        const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url);
+        const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url, providerType);
         if (extMime !== null) {
           out.push({
             type: 'text',
-            text: buildUnsupportedImageNotice(extMime, part.imageUrl.url),
+            text: buildUnsupportedImageNotice(extMime, part.imageUrl.url, providerType),
           });
           continue;
         }
@@ -296,8 +304,11 @@ export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart
         parsed.mimeType,
         decodeBase64Prefix(parsed.base64),
       );
-      if (!isModelAcceptedImageMime(effectiveMime)) {
-        out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) });
+      if (!isModelAcceptedImageMime(effectiveMime, providerType)) {
+        out.push({
+          type: 'text',
+          text: buildUnsupportedImageNotice(effectiveMime, undefined, providerType),
+        });
         continue;
       }
       const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`;
@@ -313,12 +324,15 @@ export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart
 
 export async function compressImageContentParts(
   parts: readonly ContentPart[],
-  options: CompressImageOptions & { readonly annotate?: CompressAnnotateOptions } = {},
+  options: CompressImageOptions & {
+    readonly annotate?: CompressAnnotateOptions;
+    readonly providerType?: string;
+  } = {},
 ): Promise<CompressedContentParts> {
-  const { annotate, ...compressOptions } = options;
+  const { annotate, providerType, ...compressOptions } = options;
   const out: ContentPart[] = [];
   const captions: string[] = [];
-  for (const part of gateImageFormatParts(parts)) {
+  for (const part of gateImageFormatParts(parts, providerType)) {
     if (part.type === 'image_url') {
       const parsed = parseImageDataUrl(part.imageUrl.url);
       if (parsed !== null) {