feat(agent-core-v2): read UTF-16 text files by transcoding to UTF-8

.changeset/read-utf16-text-files.md

@@ -0,0 +1,5 @@
+---
+"@moonshot-ai/kimi-code": patch
+---
+
+Read UTF-16 LE/BE text files (with or without a BOM) by transcoding them to UTF-8 instead of refusing them as binary; the web UI file viewer displays them as text as well.

packages/agent-core-v2/src/_base/text/encoding.ts

@@ -0,0 +1,110 @@
+/**
+ * `_base` text helpers — UTF text encoding detection and decoding.
+ *
+ * Detection algorithm derived from VS Code
+ * `src/vs/workbench/services/textfile/common/encoding.ts`
+ * (MIT License, Copyright (c) Microsoft Corporation): BOM sniffing plus a
+ * zero-byte parity heuristic that recognizes BOM-less UTF-16 LE/BE, so text
+ * files saved as UTF-16 (e.g. Windows Notepad `.txt`) can be transcoded to
+ * UTF-8 instead of being refused as binary.
+ *
+ * The parity heuristic deliberately deviates from VS Code in one way: VS
+ * Code requires *every* byte pair to conform (a single CJK character, whose
+ * UTF-16 unit carries no zero byte, falsifies the pattern and the file is
+ * deemed binary). Here, zero bytes must instead appear at least twice and at
+ * exactly one parity — odd indices mean UTF-16 LE (`0xAA 0x00`), even
+ * indices mean UTF-16 BE (`0x00 0xAA`) — which tolerates mixed Latin/CJK
+ * content while still rejecting real binaries (zeros at both parities, or
+ * an isolated zero byte). Legacy 8-bit encodings (GBK, Big5, Shift-JIS, …)
+ * are never guessed — a wrong silent guess is worse than a clear refusal.
+ *
+ * Pure functions over bytes; no io happens here.
+ */
+
+export type UtfTextEncoding = 'utf-8' | 'utf-16le' | 'utf-16be';
+
+export interface TextEncodingDetection {
+  /**
+   * Detected encoding. `'utf-8'` when no signal points elsewhere (also the
+   * placeholder when `seemsBinary` is true).
+   */
+  readonly encoding: UtfTextEncoding;
+  /**
+   * True when zero bytes appear but fit neither UTF-16 pattern — the sample
+   * should be treated as binary, not text.
+   */
+  readonly seemsBinary: boolean;
+}
+
+/** Number of leading bytes inspected for the zero-byte heuristic. */
+export const ENCODING_DETECTION_SAMPLE_BYTES = 512;
+
+/**
+ * Minimum zero bytes (at a single parity) before the BOM-less UTF-16
+ * heuristic commits. One isolated zero byte is too ambiguous — a short
+ * binary blob like `"plain prefix" + 00 01` would otherwise masquerade as
+ * UTF-16 BE.
+ */
+const MIN_ZERO_BYTES_FOR_UTF16 = 2;
+
+const UTF16BE_BOM = [0xfe, 0xff] as const;
+const UTF16LE_BOM = [0xff, 0xfe] as const;
+const UTF8_BOM = [0xef, 0xbb, 0xbf] as const;
+
+/**
+ * Detect the encoding of a text file from its leading bytes.
+ *
+ * Known limitation inherited from the reference implementation: a BOM-less
+ * UTF-16 file whose content carries no zero bytes at all (e.g. purely CJK
+ * text) is reported as `'utf-8'`; strict UTF-8 decoding of it will then fail
+ * or produce garbage. Notepad and most editors write a BOM, so this is rare
+ * in practice.
+ */
+export function detectTextEncoding(sample: Uint8Array): TextEncodingDetection {
+  // Always trust a BOM first.
+  if (sample.length >= 2) {
+    const b0 = sample[0]!;
+    const b1 = sample[1]!;
+    if (b0 === UTF16BE_BOM[0] && b1 === UTF16BE_BOM[1]) {
+      return { encoding: 'utf-16be', seemsBinary: false };
+    }
+    if (b0 === UTF16LE_BOM[0] && b1 === UTF16LE_BOM[1]) {
+      return { encoding: 'utf-16le', seemsBinary: false };
+    }
+    if (sample.length >= 3 && b0 === UTF8_BOM[0] && b1 === UTF8_BOM[1] && sample[2] === UTF8_BOM[2]) {
+      return { encoding: 'utf-8', seemsBinary: false };
+    }
+  }
+
+  // BOM-less UTF-16: zero bytes cluster at one parity — odd indices for LE
+  // (`0xAA 0x00`), even for BE (`0x00 0xAA`). CJK units carry no zero byte,
+  // so only the *placement* of zeros is checked, not their density. Zeros
+  // at both parities, or fewer than the ambiguity threshold, mean binary.
+  let zerosAtOdd = 0;
+  let zerosAtEven = 0;
+  const limit = Math.min(sample.length, ENCODING_DETECTION_SAMPLE_BYTES);
+  for (let i = 0; i < limit; i++) {
+    if (sample[i] !== 0) continue;
+    if (i % 2 === 1) zerosAtOdd++;
+    else zerosAtEven++;
+  }
+
+  if (zerosAtOdd === 0 && zerosAtEven === 0) {
+    return { encoding: 'utf-8', seemsBinary: false };
+  }
+  if (zerosAtEven === 0 && zerosAtOdd >= MIN_ZERO_BYTES_FOR_UTF16) {
+    return { encoding: 'utf-16le', seemsBinary: false };
+  }
+  if (zerosAtOdd === 0 && zerosAtEven >= MIN_ZERO_BYTES_FOR_UTF16) {
+    return { encoding: 'utf-16be', seemsBinary: false };
+  }
+  return { encoding: 'utf-8', seemsBinary: true };
+}
+
+/**
+ * Decode bytes in a detected UTF encoding to a JS string. Malformed
+ * sequences are replaced (non-fatal) and a leading BOM is stripped.
+ */
+export function decodeUtfText(bytes: Uint8Array, encoding: UtfTextEncoding): string {
+  return new TextDecoder(encoding, { fatal: false }).decode(bytes);
+}

packages/agent-core-v2/src/_base/text/line-endings.ts

@@ -56,3 +56,24 @@ export function materializeModelText(text: string, lineEndingStyle: LineEndingSt
 export function makeCarriageReturnsVisible(text: string): string {
   return text.replaceAll('\r', '\\r');
 }
+
+/**
+ * Split text into lines, keeping each line's trailing `\n` (the final line
+ * may lack one). Same semantics as Python's `str.splitlines(keepends=True)`
+ * restricted to `\n` boundaries.
+ */
+export function splitLinesKeepingTerminator(text: string): string[] {
+  if (text.length === 0) return [];
+  const lines: string[] = [];
+  let start = 0;
+  for (let i = 0; i < text.length; i += 1) {
+    if (text.codePointAt(i) === 0x0a) {
+      lines.push(text.slice(start, i + 1));
+      start = i + 1;
+    }
+  }
+  if (start < text.length) {
+    lines.push(text.slice(start));
+  }
+  return lines;
+}

packages/agent-core-v2/src/agent/tools/os/read/read.md

@@ -8,7 +8,7 @@ When you need several files, prefer to read them in parallel: emit multiple `Rea
 - Returns up to ${MAX_LINES} lines or ${MAX_BYTES_KB} KB per call, whichever comes first; lines longer than ${MAX_LINE_LENGTH} chars are truncated mid-line.
 - Page larger files with `line_offset` (1-based start line) and `n_lines`. Omit `n_lines` to read up to the ${MAX_LINES}-line cap.
 - Sensitive files (`.env` files, credential stores, SSH private keys, and similar secrets) are refused to protect secrets; do not attempt to read them. Templates and public keys are exempt: `.env.example` / `.env.sample` / `.env.template` and public SSH keys such as `id_rsa.pub` read normally.
-- Only UTF-8 text files can be read. Non-UTF-8 encodings, binary files, and files containing NUL bytes are refused; use `ReadMediaFile` for images or video, and Bash or an MCP tool for other binary formats.
+- UTF-8 text files are read directly. UTF-16 LE/BE text files (with or without a BOM) are detected automatically and transcoded to UTF-8 for display; the status block notes the detected encoding, and Edit/Write on such a file still expect UTF-8 — convert its encoding first (e.g. `iconv` via Bash). Other encodings (e.g. GBK), binary files, and files containing NUL bytes are refused; use `ReadMediaFile` for images or video, and Bash or an MCP tool for other binary formats.
 - Negative line_offset reads from the end of the file (for example, -100 reads the last 100 lines); the absolute value cannot exceed ${MAX_LINES}.
 - Output format: `<line-number>\t<content>` per line.
 - A `<system>...</system>` status block is appended after the file content; it summarizes how much was read (line and byte counts, truncation, line-ending notes) and is not part of the file itself.

packages/agent-core-v2/src/agent/tools/os/read/read.ts

@@ -9,8 +9,11 @@
  * are displayed with LF line endings; mixed or lone carriage returns are
  * shown as `\r` so the model can reproduce them exactly.
  *
- * Binary, non-UTF-8, NUL-containing, image and video files are refused;
- * images/videos are redirected to ReadMediaFile. Supports one-based
+ * UTF-16 LE/BE text files (with a BOM, or recognized via the zero-byte
+ * parity heuristic) are transparently transcoded to UTF-8 for display, up to
+ * `TRANSCODE_MAX_BYTES`. Binary, other non-UTF encodings, NUL-containing,
+ * image and video files are refused; images/videos are redirected to
+ * ReadMediaFile. Supports one-based
  * `line_offset` / `n_lines` pagination and a negative `line_offset` tail
  * mode, bounded by the per-call caps owned here (`MAX_LINES`,
  * `MAX_LINE_LENGTH`, `MAX_BYTES`).
@@ -27,6 +30,13 @@ export const MAX_LINES: number = 1000;
 export const MAX_LINE_LENGTH: number = 2000;
 export const MAX_BYTES: number = 100 * 1024;
 
+/**
+ * Largest file the Read tool transcodes from UTF-16 in memory. Unlike the
+ * streaming UTF-8 path, transcoding needs the whole file decoded at once;
+ * 10 MiB mirrors kap-server's `FS_READ_MAX_BYTES`.
+ */
+export const TRANSCODE_MAX_BYTES: number = 10 * 1024 * 1024;
+
 const PositiveLineOffsetSchema = z.number().int().min(1);
 const TailLineOffsetSchema = z.number().int().min(-MAX_LINES).max(-1);