diff --git a/.changeset/read-utf16-text-files.md b/.changeset/read-utf16-text-files.md new file mode 100644 index 0000000000..aa703e466a --- /dev/null +++ b/.changeset/read-utf16-text-files.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Read UTF-16 LE/BE text files (with or without a BOM) by transcoding them to UTF-8 instead of refusing them as binary; the web UI file viewer displays them as text as well. diff --git a/packages/agent-core-v2/src/_base/text/encoding.ts b/packages/agent-core-v2/src/_base/text/encoding.ts new file mode 100644 index 0000000000..714f05251d --- /dev/null +++ b/packages/agent-core-v2/src/_base/text/encoding.ts @@ -0,0 +1,110 @@ +/** + * `_base` text helpers — UTF text encoding detection and decoding. + * + * Detection algorithm derived from VS Code + * `src/vs/workbench/services/textfile/common/encoding.ts` + * (MIT License, Copyright (c) Microsoft Corporation): BOM sniffing plus a + * zero-byte parity heuristic that recognizes BOM-less UTF-16 LE/BE, so text + * files saved as UTF-16 (e.g. Windows Notepad `.txt`) can be transcoded to + * UTF-8 instead of being refused as binary. + * + * The parity heuristic deliberately deviates from VS Code in one way: VS + * Code requires *every* byte pair to conform (a single CJK character, whose + * UTF-16 unit carries no zero byte, falsifies the pattern and the file is + * deemed binary). Here, zero bytes must instead appear at least twice and at + * exactly one parity — odd indices mean UTF-16 LE (`0xAA 0x00`), even + * indices mean UTF-16 BE (`0x00 0xAA`) — which tolerates mixed Latin/CJK + * content while still rejecting real binaries (zeros at both parities, or + * an isolated zero byte). Legacy 8-bit encodings (GBK, Big5, Shift-JIS, …) + * are never guessed — a wrong silent guess is worse than a clear refusal. + * + * Pure functions over bytes; no io happens here. + */ + +export type UtfTextEncoding = 'utf-8' | 'utf-16le' | 'utf-16be'; + +export interface TextEncodingDetection { + /** + * Detected encoding. `'utf-8'` when no signal points elsewhere (also the + * placeholder when `seemsBinary` is true). + */ + readonly encoding: UtfTextEncoding; + /** + * True when zero bytes appear but fit neither UTF-16 pattern — the sample + * should be treated as binary, not text. + */ + readonly seemsBinary: boolean; +} + +/** Number of leading bytes inspected for the zero-byte heuristic. */ +export const ENCODING_DETECTION_SAMPLE_BYTES = 512; + +/** + * Minimum zero bytes (at a single parity) before the BOM-less UTF-16 + * heuristic commits. One isolated zero byte is too ambiguous — a short + * binary blob like `"plain prefix" + 00 01` would otherwise masquerade as + * UTF-16 BE. + */ +const MIN_ZERO_BYTES_FOR_UTF16 = 2; + +const UTF16BE_BOM = [0xfe, 0xff] as const; +const UTF16LE_BOM = [0xff, 0xfe] as const; +const UTF8_BOM = [0xef, 0xbb, 0xbf] as const; + +/** + * Detect the encoding of a text file from its leading bytes. + * + * Known limitation inherited from the reference implementation: a BOM-less + * UTF-16 file whose content carries no zero bytes at all (e.g. purely CJK + * text) is reported as `'utf-8'`; strict UTF-8 decoding of it will then fail + * or produce garbage. Notepad and most editors write a BOM, so this is rare + * in practice. + */ +export function detectTextEncoding(sample: Uint8Array): TextEncodingDetection { + // Always trust a BOM first. + if (sample.length >= 2) { + const b0 = sample[0]!; + const b1 = sample[1]!; + if (b0 === UTF16BE_BOM[0] && b1 === UTF16BE_BOM[1]) { + return { encoding: 'utf-16be', seemsBinary: false }; + } + if (b0 === UTF16LE_BOM[0] && b1 === UTF16LE_BOM[1]) { + return { encoding: 'utf-16le', seemsBinary: false }; + } + if (sample.length >= 3 && b0 === UTF8_BOM[0] && b1 === UTF8_BOM[1] && sample[2] === UTF8_BOM[2]) { + return { encoding: 'utf-8', seemsBinary: false }; + } + } + + // BOM-less UTF-16: zero bytes cluster at one parity — odd indices for LE + // (`0xAA 0x00`), even for BE (`0x00 0xAA`). CJK units carry no zero byte, + // so only the *placement* of zeros is checked, not their density. Zeros + // at both parities, or fewer than the ambiguity threshold, mean binary. + let zerosAtOdd = 0; + let zerosAtEven = 0; + const limit = Math.min(sample.length, ENCODING_DETECTION_SAMPLE_BYTES); + for (let i = 0; i < limit; i++) { + if (sample[i] !== 0) continue; + if (i % 2 === 1) zerosAtOdd++; + else zerosAtEven++; + } + + if (zerosAtOdd === 0 && zerosAtEven === 0) { + return { encoding: 'utf-8', seemsBinary: false }; + } + if (zerosAtEven === 0 && zerosAtOdd >= MIN_ZERO_BYTES_FOR_UTF16) { + return { encoding: 'utf-16le', seemsBinary: false }; + } + if (zerosAtOdd === 0 && zerosAtEven >= MIN_ZERO_BYTES_FOR_UTF16) { + return { encoding: 'utf-16be', seemsBinary: false }; + } + return { encoding: 'utf-8', seemsBinary: true }; +} + +/** + * Decode bytes in a detected UTF encoding to a JS string. Malformed + * sequences are replaced (non-fatal) and a leading BOM is stripped. + */ +export function decodeUtfText(bytes: Uint8Array, encoding: UtfTextEncoding): string { + return new TextDecoder(encoding, { fatal: false }).decode(bytes); +} diff --git a/packages/agent-core-v2/src/_base/text/line-endings.ts b/packages/agent-core-v2/src/_base/text/line-endings.ts index 71f4367084..3f27470a8a 100644 --- a/packages/agent-core-v2/src/_base/text/line-endings.ts +++ b/packages/agent-core-v2/src/_base/text/line-endings.ts @@ -56,3 +56,24 @@ export function materializeModelText(text: string, lineEndingStyle: LineEndingSt export function makeCarriageReturnsVisible(text: string): string { return text.replaceAll('\r', '\\r'); } + +/** + * Split text into lines, keeping each line's trailing `\n` (the final line + * may lack one). Same semantics as Python's `str.splitlines(keepends=True)` + * restricted to `\n` boundaries. + */ +export function splitLinesKeepingTerminator(text: string): string[] { + if (text.length === 0) return []; + const lines: string[] = []; + let start = 0; + for (let i = 0; i < text.length; i += 1) { + if (text.codePointAt(i) === 0x0a) { + lines.push(text.slice(start, i + 1)); + start = i + 1; + } + } + if (start < text.length) { + lines.push(text.slice(start)); + } + return lines; +} diff --git a/packages/agent-core-v2/src/agent/tools/os/read/read.md b/packages/agent-core-v2/src/agent/tools/os/read/read.md index 6df4c64f99..8cfab273b0 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/read.md +++ b/packages/agent-core-v2/src/agent/tools/os/read/read.md @@ -8,7 +8,7 @@ When you need several files, prefer to read them in parallel: emit multiple `Rea - Returns up to ${MAX_LINES} lines or ${MAX_BYTES_KB} KB per call, whichever comes first; lines longer than ${MAX_LINE_LENGTH} chars are truncated mid-line. - Page larger files with `line_offset` (1-based start line) and `n_lines`. Omit `n_lines` to read up to the ${MAX_LINES}-line cap. - Sensitive files (`.env` files, credential stores, SSH private keys, and similar secrets) are refused to protect secrets; do not attempt to read them. Templates and public keys are exempt: `.env.example` / `.env.sample` / `.env.template` and public SSH keys such as `id_rsa.pub` read normally. -- Only UTF-8 text files can be read. Non-UTF-8 encodings, binary files, and files containing NUL bytes are refused; use `ReadMediaFile` for images or video, and Bash or an MCP tool for other binary formats. +- UTF-8 text files are read directly. UTF-16 LE/BE text files (with or without a BOM) are detected automatically and transcoded to UTF-8 for display; the status block notes the detected encoding, and Edit/Write on such a file still expect UTF-8 — convert its encoding first (e.g. `iconv` via Bash). Other encodings (e.g. GBK), binary files, and files containing NUL bytes are refused; use `ReadMediaFile` for images or video, and Bash or an MCP tool for other binary formats. - Negative line_offset reads from the end of the file (for example, -100 reads the last 100 lines); the absolute value cannot exceed ${MAX_LINES}. - Output format: `\t` per line. - A `...` status block is appended after the file content; it summarizes how much was read (line and byte counts, truncation, line-ending notes) and is not part of the file itself. diff --git a/packages/agent-core-v2/src/agent/tools/os/read/read.ts b/packages/agent-core-v2/src/agent/tools/os/read/read.ts index 59d88ea6f2..9f4b1dea5f 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/read.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/read.ts @@ -9,8 +9,11 @@ * are displayed with LF line endings; mixed or lone carriage returns are * shown as `\r` so the model can reproduce them exactly. * - * Binary, non-UTF-8, NUL-containing, image and video files are refused; - * images/videos are redirected to ReadMediaFile. Supports one-based + * UTF-16 LE/BE text files (with a BOM, or recognized via the zero-byte + * parity heuristic) are transparently transcoded to UTF-8 for display, up to + * `TRANSCODE_MAX_BYTES`. Binary, other non-UTF encodings, NUL-containing, + * image and video files are refused; images/videos are redirected to + * ReadMediaFile. Supports one-based * `line_offset` / `n_lines` pagination and a negative `line_offset` tail * mode, bounded by the per-call caps owned here (`MAX_LINES`, * `MAX_LINE_LENGTH`, `MAX_BYTES`). @@ -27,6 +30,13 @@ export const MAX_LINES: number = 1000; export const MAX_LINE_LENGTH: number = 2000; export const MAX_BYTES: number = 100 * 1024; +/** + * Largest file the Read tool transcodes from UTF-16 in memory. Unlike the + * streaming UTF-8 path, transcoding needs the whole file decoded at once; + * 10 MiB mirrors kap-server's `FS_READ_MAX_BYTES`. + */ +export const TRANSCODE_MAX_BYTES: number = 10 * 1024 * 1024; + const PositiveLineOffsetSchema = z.number().int().min(1); const TailLineOffsetSchema = z.number().int().min(-MAX_LINES).max(-1); diff --git a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts index d9752a4843..9d4b5ba491 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts @@ -5,7 +5,9 @@ * line/byte budgets from the contract, normalizes line endings for display * (pure CRLF shown as LF, mixed or lone carriage returns made visible as * `\r`), refuses binary / media files up front, and composes the `` - * finish note on the `note` side channel. + * finish note on the `note` side channel. UTF-16 LE/BE text (with a BOM or + * the zero-byte parity heuristic) is decoded whole via `readBytes` and + * transcoded to UTF-8, bounded by `TRANSCODE_MAX_BYTES`. * * Path safety goes through the shared path access resolver used by * Read/Write/Edit. Read access flows through the os `hostFs` domain @@ -39,7 +41,8 @@ import { import { MEDIA_SNIFF_BYTES, detectFileType } from '#/agent/media/file-type'; import { toInputJsonSchema } from '#/tool/input-schema'; import { literalRulePattern, matchesPathRuleSubject } from '#/tool/rule-match'; -import { makeCarriageReturnsVisible, type LineEndingStyle } from '#/_base/text/line-endings'; +import { makeCarriageReturnsVisible, splitLinesKeepingTerminator, type LineEndingStyle } from '#/_base/text/line-endings'; +import { decodeUtfText, detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; import { renderPrompt } from '#/_base/utils/render-prompt'; import { IReadTool, @@ -47,6 +50,7 @@ import { MAX_LINE_LENGTH, MAX_LINES, ReadInputSchema, + TRANSCODE_MAX_BYTES, type ReadInput, } from './read'; import readDescriptionTemplate from './read.md?raw'; @@ -76,6 +80,7 @@ interface FinishReadResultInput { readonly startLine: number; readonly totalLines: number; readonly requestedLines: number; + readonly detectedEncoding?: UtfTextEncoding; } function truncateLine(line: string, maxLength: number): string { @@ -184,6 +189,21 @@ function containsNulByte(text: string): boolean { return text.includes('\u0000'); } +function encodingDisplayName(encoding: UtfTextEncoding): string { + switch (encoding) { + case 'utf-16le': + return 'UTF-16 LE'; + case 'utf-16be': + return 'UTF-16 BE'; + default: + return 'UTF-8'; + } +} + +async function* decodedLines(lines: readonly string[]): AsyncGenerator { + yield* lines; +} + function notReadableFileOutput(path: string): string { return ( `"${path}" is not readable as UTF-8 text. ` + @@ -192,6 +212,14 @@ function notReadableFileOutput(path: string): string { ); } +function notUtf8DecodableFileOutput(path: string): string { + return ( + `"${path}" is not valid UTF-8 or UTF-16 text. ` + + 'Only UTF-8 and UTF-16 text files can be read; ' + + 'for other encodings (e.g. GBK), convert the file to UTF-8 first (e.g. `iconv` via Bash).' + ); +} + const READ_DESCRIPTION = renderPrompt(readDescriptionTemplate, { MAX_LINES, MAX_BYTES_KB: MAX_BYTES / 1024, @@ -265,11 +293,35 @@ export class ReadTool implements IReadTool { output: `"${args.path}" is a ${fileType.kind} file. Use ReadMediaFile to read image or video files.`, }; } - if (fileType.kind === 'unknown') { + + // A BOM marks UTF-16 even when the header carries no NUL bytes (e.g. + // CJK-only content reads as printable ASCII), so detect the encoding + // before falling through to the strict UTF-8 text path. + const detection = detectTextEncoding(header); + let lines: AsyncIterable; + let detectedEncoding: UtfTextEncoding | undefined; + if (!detection.seemsBinary && detection.encoding !== 'utf-8') { + // UTF-16 LE/BE text (BOM or zero-byte parity heuristic): decode the + // whole file and transcode to UTF-8 for display. + if (stat.size > TRANSCODE_MAX_BYTES) { + return { + isError: true, + output: + `"${args.path}" is ${encodingDisplayName(detection.encoding)} text but too large to transcode ` + + `(${String(stat.size)} bytes > ${String(TRANSCODE_MAX_BYTES)}). ` + + 'Convert it to UTF-8 first (e.g. `iconv` via Bash).', + }; + } + const decoded = decodeUtfText(await this.fs.readBytes(safePath), detection.encoding); + detectedEncoding = detection.encoding; + lines = decodedLines(splitLinesKeepingTerminator(decoded)); + } else if (fileType.kind === 'unknown') { return { isError: true, output: notReadableFileOutput(args.path), }; + } else { + lines = this.fs.readLines(safePath, { errors: 'strict' }); } const lineOffset = args.line_offset ?? 1; @@ -278,23 +330,25 @@ export class ReadTool implements IReadTool { if (lineOffset < 0) { return await this.readTail( - safePath, args.path, + lines, lineOffset, effectiveLimit, requestedLines, + detectedEncoding, ); } return await this.readForward( - safePath, args.path, + lines, lineOffset, effectiveLimit, requestedLines, + detectedEncoding, ); } catch (error) { if (isTextDecodeError(error)) { - return { isError: true, output: notReadableFileOutput(args.path) }; + return { isError: true, output: notUtf8DecodableFileOutput(args.path) }; } return { isError: true, @@ -304,11 +358,12 @@ export class ReadTool implements IReadTool { } private async readForward( - safePath: string, displayPath: string, + lines: AsyncIterable, lineOffset: number, effectiveLimit: number, requestedLines: number, + detectedEncoding?: UtfTextEncoding, ): Promise { const selectedEntries: ReadLineEntry[] = []; const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; @@ -316,7 +371,7 @@ export class ReadTool implements IReadTool { let maxLinesReached = false; let collectionClosed = false; - for await (const rawLine of this.fs.readLines(safePath, { errors: 'strict' })) { + for await (const rawLine of lines) { if (containsNulByte(rawLine)) { return { isError: true, output: notReadableFileOutput(displayPath) }; } @@ -357,22 +412,24 @@ export class ReadTool implements IReadTool { startLine: selectedEntries.length > 0 ? lineOffset : 0, totalLines: currentLineNo, requestedLines, + detectedEncoding, }); } private async readTail( - safePath: string, displayPath: string, + lines: AsyncIterable, lineOffset: number, effectiveLimit: number, requestedLines: number, + detectedEncoding?: UtfTextEncoding, ): Promise { const tailCount = Math.abs(lineOffset); const entries: ReadLineEntry[] = []; const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; let currentLineNo = 0; - for await (const rawLine of this.fs.readLines(safePath, { errors: 'strict' })) { + for await (const rawLine of lines) { if (containsNulByte(rawLine)) { return { isError: true, output: notReadableFileOutput(displayPath) }; } @@ -393,6 +450,7 @@ export class ReadTool implements IReadTool { effectiveLimit, totalLines: currentLineNo, requestedLines, + detectedEncoding, }); } @@ -402,6 +460,7 @@ export class ReadTool implements IReadTool { effectiveLimit: number; totalLines: number; requestedLines: number; + detectedEncoding?: UtfTextEncoding; }): ExecutableToolResult { const lineEndingStyle = lineEndingStyleFromFlags(input.lineEndingFlags); let renderedCandidates = input.entries.slice(0, input.effectiveLimit).map((entry) => { @@ -447,6 +506,7 @@ export class ReadTool implements IReadTool { startLine: renderedCandidates[0]?.entry.lineNo ?? 0, totalLines: input.totalLines, requestedLines: input.requestedLines, + detectedEncoding: input.detectedEncoding, }); } @@ -483,6 +543,11 @@ export class ReadTool implements IReadTool { 'Mixed or lone carriage-return line endings are shown as \\r. Use exact \\r\\n or \\r escapes in Edit.old_string for those lines.', ); } + if (input.detectedEncoding !== undefined) { + parts.push( + `Detected file encoding: ${encodingDisplayName(input.detectedEncoding)}; content transcoded to UTF-8 for display. Edit and Write expect UTF-8 — convert the file's encoding first (e.g. \`iconv\` via Bash).`, + ); + } return parts.join(' '); } } diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index 772f8cb3fe..ac47a89cc4 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -59,6 +59,7 @@ const FsWireErrorCode = { import ignore, { type Ignore } from 'ignore'; import { LifecycleScope, ScopeActivation, registerScopedService } from '#/_base/di/scope'; +import { decodeUtfText, detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; import { buildEtag, countLines, @@ -255,7 +256,20 @@ export class WorkspaceFsService implements IWorkspaceFsService { const sampleSize = Math.min(FS_BINARY_SAMPLE_BYTES, st.size); const sample = sampleSize === 0 ? new Uint8Array() : await this.hostFs.readBytes(abs, sampleSize); - const isBinary = detectBinary(sample); + let isBinary = detectBinary(sample); + + // Trust encoding detection over the binary heuristic: a binary-looking + // sample can still be UTF-16 LE/BE text, and a BOM-marked UTF-16 file + // may not look binary at all (CJK-only content carries no zero bytes). + // Both are transcoded to UTF-8 so text clients can display them. + let transcodeEncoding: UtfTextEncoding | undefined; + if (req.encoding !== 'base64') { + const detection = detectTextEncoding(sample); + if (!detection.seemsBinary && detection.encoding !== 'utf-8') { + transcodeEncoding = detection.encoding; + isBinary = false; + } + } if (isBinary && req.encoding === 'utf-8') { throw new Error2(ErrorCodes.FS_IS_BINARY, `file is binary: ${req.path}`, { @@ -263,10 +277,24 @@ export class WorkspaceFsService implements IWorkspaceFsService { }); } - const effectiveLength = Math.min(req.length, st.size - req.offset); + // When transcoding, the offset/length window applies to the decoded + // UTF-8 bytes — the representation the client actually paginates over. + let totalLength = st.size; + let decodedBytes: Uint8Array | undefined; + if (transcodeEncoding !== undefined) { + decodedBytes = Buffer.from( + decodeUtfText(await this.hostFs.readBytes(abs), transcodeEncoding), + 'utf-8', + ); + totalLength = decodedBytes.length; + } + + const effectiveLength = Math.min(req.length, totalLength - req.offset); let bytes: Uint8Array; if (effectiveLength <= 0) { bytes = new Uint8Array(); + } else if (decodedBytes !== undefined) { + bytes = decodedBytes.subarray(req.offset, req.offset + effectiveLength); } else { const window = await this.hostFs.readBytes(abs, req.offset + effectiveLength); bytes = window.subarray(req.offset, req.offset + effectiveLength); @@ -278,7 +306,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { encoding === 'utf-8' ? Buffer.from(bytes).toString('utf-8') : Buffer.from(bytes).toString('base64'); - const truncated = req.offset + effectiveLength < st.size; + const truncated = req.offset + effectiveLength < totalLength; const out: FsReadResponse = { path: rel, diff --git a/packages/agent-core-v2/test/_base/text/encoding.test.ts b/packages/agent-core-v2/test/_base/text/encoding.test.ts new file mode 100644 index 0000000000..1457b165a2 --- /dev/null +++ b/packages/agent-core-v2/test/_base/text/encoding.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest'; + +import { + decodeUtfText, + detectTextEncoding, + ENCODING_DETECTION_SAMPLE_BYTES, +} from '#/_base/text/encoding'; +import { splitLinesKeepingTerminator } from '#/_base/text/line-endings'; + +function utf16Le(text: string): Buffer { + return Buffer.from(text, 'utf16le'); +} + +function utf16Be(text: string): Buffer { + const le = utf16Le(text); + const be = Buffer.alloc(le.length); + for (let i = 0; i < le.length; i += 2) { + be[i] = le[i + 1]!; + be[i + 1] = le[i]!; + } + return be; +} + +describe('detectTextEncoding', () => { + it('detects encodings by BOM', () => { + expect(detectTextEncoding(Buffer.from([0xef, 0xbb, 0xbf, 0x61])).encoding).toBe('utf-8'); + expect(detectTextEncoding(Buffer.from([0xff, 0xfe, 0x61, 0x00])).encoding).toBe('utf-16le'); + expect(detectTextEncoding(Buffer.from([0xfe, 0xff, 0x00, 0x61])).encoding).toBe('utf-16be'); + }); + + it('trusts the BOM even when the sample carries no zero bytes (CJK-only)', () => { + const le = Buffer.concat([Buffer.from([0xff, 0xfe]), utf16Le('你好世界')]); + expect(detectTextEncoding(le)).toEqual({ encoding: 'utf-16le', seemsBinary: false }); + const be = Buffer.concat([Buffer.from([0xfe, 0xff]), utf16Be('你好世界')]); + expect(detectTextEncoding(be)).toEqual({ encoding: 'utf-16be', seemsBinary: false }); + }); + + it('detects BOM-less UTF-16 by the zero-byte parity heuristic', () => { + expect(detectTextEncoding(utf16Le('hello world, plain ascii')).encoding).toBe('utf-16le'); + expect(detectTextEncoding(utf16Be('hello world, plain ascii')).encoding).toBe('utf-16be'); + }); + + it('tolerates CJK characters in BOM-less UTF-16 (their units carry no zero byte)', () => { + expect(detectTextEncoding(utf16Le('hello 你好\nsecond line')).encoding).toBe('utf-16le'); + expect(detectTextEncoding(utf16Be('hello 你好\nsecond line')).encoding).toBe('utf-16be'); + }); + + it('reports BOM-less UTF-16 with no zero bytes at all as utf-8 (known limitation)', () => { + // Pure CJK content has no zero bytes in UTF-16 — undetectable without + // statistical guessing, same as VS Code. + expect(detectTextEncoding(utf16Le('你好世界')).encoding).toBe('utf-8'); + }); + + it('treats an isolated zero byte as binary (too ambiguous)', () => { + expect(detectTextEncoding(Buffer.from([0x61, 0x00])).seemsBinary).toBe(true); + expect(detectTextEncoding(Buffer.from([0x00, 0x61])).seemsBinary).toBe(true); + }); + + it('limits the zero-byte heuristic to the leading sample window', () => { + const sample = Buffer.alloc(ENCODING_DETECTION_SAMPLE_BYTES + 2, 0x61); + sample[ENCODING_DETECTION_SAMPLE_BYTES + 1] = 0x00; + expect(detectTextEncoding(sample)).toEqual({ encoding: 'utf-8', seemsBinary: false }); + }); + + it('flags zero bytes at both parities as binary', () => { + expect(detectTextEncoding(Buffer.from([0x00, 0x00, 0x61, 0x62])).seemsBinary).toBe(true); + const prefix = Buffer.concat([Buffer.from('plain prefix'), Buffer.from([0x00, 0x01])]); + expect(detectTextEncoding(prefix).seemsBinary).toBe(true); + }); + + it('treats plain ASCII / UTF-8 and empty samples as utf-8 text', () => { + expect(detectTextEncoding(new Uint8Array())).toEqual({ encoding: 'utf-8', seemsBinary: false }); + expect(detectTextEncoding(Buffer.from('plain ascii\n')).seemsBinary).toBe(false); + expect(detectTextEncoding(Buffer.from('中文内容\n', 'utf8'))).toEqual({ + encoding: 'utf-8', + seemsBinary: false, + }); + }); +}); + +describe('decodeUtfText', () => { + it('decodes UTF-16 LE/BE and strips the BOM', () => { + const le = Buffer.concat([Buffer.from([0xff, 0xfe]), utf16Le('你好\nworld')]); + expect(decodeUtfText(le, 'utf-16le')).toBe('你好\nworld'); + const be = Buffer.concat([Buffer.from([0xfe, 0xff]), utf16Be('你好\nworld')]); + expect(decodeUtfText(be, 'utf-16be')).toBe('你好\nworld'); + }); + + it('decodes UTF-8 and strips the BOM', () => { + const bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('text', 'utf8')]); + expect(decodeUtfText(bytes, 'utf-8')).toBe('text'); + }); + + it('replaces malformed sequences instead of throwing', () => { + expect(decodeUtfText(Buffer.from([0xff]), 'utf-16le')).toBe('�'); + }); +}); + +describe('splitLinesKeepingTerminator', () => { + it('keeps line terminators and the unterminated tail', () => { + expect(splitLinesKeepingTerminator('a\nb\n')).toEqual(['a\n', 'b\n']); + expect(splitLinesKeepingTerminator('a\nb')).toEqual(['a\n', 'b']); + expect(splitLinesKeepingTerminator('')).toEqual([]); + expect(splitLinesKeepingTerminator('\n')).toEqual(['\n']); + }); +}); diff --git a/packages/agent-core-v2/test/agent/loop/loop.test.ts b/packages/agent-core-v2/test/agent/loop/loop.test.ts index f9d1416c44..16d7be35f3 100644 --- a/packages/agent-core-v2/test/agent/loop/loop.test.ts +++ b/packages/agent-core-v2/test/agent/loop/loop.test.ts @@ -121,8 +121,8 @@ describe('Agent loop', () => { [emit] turn.step.started { "turnId": 0, "step": 1, "stepId": "" } [emit] agent.activity.updated { "lifecycle": "ready", "turn": { "turnId": 0, "origin": { "kind": "user" }, "phase": "running", "step": 1, "ending": false, "pendingApprovals": [], "activeToolCalls": [], "since": "