From 1cf09437f9d6cf8227f28d6a85a84d4766f26bc0 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Fri, 8 May 2026 19:22:38 +0000 Subject: [PATCH 1/2] fix(cli): stream UTF-8 reads in read tool again Optimistically stream the file as UTF-8 -- the common case -- using a fatal-mode TextDecoder so the read tool can stop pulling bytes from disk once the line / 50KB byte cap is hit. Only fall back to a full-buffer iconv decode when the bytes turn out not to be valid UTF-8. The streaming + retry logic lives in a new kilo helper (packages/opencode/src/kilocode/text-stream.ts) so the read tool's `lines` function stays close to upstream OpenCode shape. --- .changeset/read-stream-utf8.md | 5 ++ packages/opencode/src/kilocode/text-stream.ts | 84 +++++++++++++++++++ packages/opencode/src/tool/read.ts | 16 ++-- 3 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 .changeset/read-stream-utf8.md create mode 100644 packages/opencode/src/kilocode/text-stream.ts diff --git a/.changeset/read-stream-utf8.md b/.changeset/read-stream-utf8.md new file mode 100644 index 00000000000..774ccbcc692 --- /dev/null +++ b/.changeset/read-stream-utf8.md @@ -0,0 +1,5 @@ +--- +"@kilocode/cli": patch +--- + +Speed up reading large files: the `read` tool now streams UTF-8 content from disk and stops once the line/byte cap is reached, instead of loading the whole file into memory first. diff --git a/packages/opencode/src/kilocode/text-stream.ts b/packages/opencode/src/kilocode/text-stream.ts new file mode 100644 index 00000000000..ae2d3de6db9 --- /dev/null +++ b/packages/opencode/src/kilocode/text-stream.ts @@ -0,0 +1,84 @@ +import { createReadStream } from "fs" +import { PassThrough, Readable } from "stream" +import * as Encoding from "./encoding" + +/** + * Encoding-aware text streaming for tools that walk a file line by line. + * + * Most files we read are UTF-8, so we stream chunks straight from disk + * (decoded strictly with `TextDecoder({ fatal: true })`) and let the consumer + * early-exit without buffering the rest of the file. Only when the bytes turn + * out not to be valid UTF-8 do we fall back to {@link Encoding.read}, which + * runs full-file detection through iconv-lite. + * + * Consumers should import this module as a namespace: + * import * as TextStream from "../kilocode/text-stream" + */ + +/** + * Sentinel error used to signal the optimistic UTF-8 stream gave up because + * the bytes are not valid UTF-8. Distinct class so {@link withFallback} can + * tell it apart from real I/O failures. + */ +export class InvalidUtf8Error extends Error { + constructor() { + super("invalid utf-8") + } +} + +/** + * UTF-8 text Readable for `filepath`. Streams chunks straight from disk so the + * caller can early-exit without buffering the rest of the file. If invalid + * UTF-8 is encountered the stream is destroyed with an {@link + * InvalidUtf8Error}. A leading UTF-8 BOM passes through as U+FEFF — the same + * behaviour as `createReadStream({ encoding: "utf8" })`. + */ +export function openUtf8(filepath: string): Readable { + const out = new PassThrough({ encoding: "utf8" }) + const raw = createReadStream(filepath) + const decoder = new TextDecoder("utf-8", { fatal: true }) + raw.on("data", (chunk) => { + try { + const text = decoder.decode(chunk as Buffer, { stream: true }) + if (text) out.write(text) + } catch { + raw.destroy() + out.destroy(new InvalidUtf8Error()) + } + }) + raw.on("end", () => { + try { + const tail = decoder.decode() + if (tail) out.write(tail) + out.end() + } catch { + out.destroy(new InvalidUtf8Error()) + } + }) + raw.on("error", (err) => out.destroy(err)) + return out +} + +/** + * Whole-file UTF-8 text Readable for `filepath`, decoded via + * {@link Encoding.read}'s detection logic. Buffers the entire decoded file + * into memory; used as a fallback for files that {@link openUtf8} rejects. + */ +export async function openDecoded(filepath: string): Promise { + const decoded = await Encoding.read(filepath) + return Readable.from([decoded.text]) +} + +/** + * Run `fn` against an optimistic UTF-8 stream of `filepath`. If the bytes + * turn out not to be valid UTF-8, `fn` is run a second time against a + * fallback iconv-decoded stream. Other errors are propagated unchanged. + */ +export async function withFallback(filepath: string, fn: (input: Readable) => Promise): Promise { + try { + return await fn(openUtf8(filepath)) + } catch (err) { + if (!(err instanceof InvalidUtf8Error)) throw err + } + return fn(await openDecoded(filepath)) +} diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index b2b8d643da9..551e3744553 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -1,9 +1,8 @@ import { lstat } from "fs/promises" // kilocode_change import { Effect, Option, Schema, Scope } from "effect" import { NonNegativeInt } from "@/util/schema" -import { createReadStream } from "fs" import * as path from "path" -import { Readable } from "stream" // kilocode_change +import type { Readable } from "stream" // kilocode_change import { createInterface } from "readline" import * as Tool from "./tool" import { AppFileSystem } from "@opencode-ai/core/filesystem" @@ -15,6 +14,7 @@ import { Instruction } from "../session/instruction" import { isPdfAttachment, sniffAttachmentMime } from "@/util/media" // kilocode_change start import * as Encoding from "../kilocode/encoding" +import * as TextStream from "../kilocode/text-stream" // kilocode_change end const DEFAULT_READ_LIMIT = 2000 @@ -354,12 +354,14 @@ export const ReadTool = Tool.define( }), ) -// kilocode_change start +// kilocode_change start - exported (so readDirectoryFiles can reuse it) and +// routed through TextStream.withFallback so non-UTF-8 files are decoded via +// iconv. The body otherwise matches upstream. export async function lines(filepath: string, opts: { limit: number; offset: number }) { - // kilocode_change end - // kilocode_change start - decode with detected encoding; replaces createReadStream(filepath, { encoding: "utf8" }) - const encoded = await Encoding.read(filepath) - const stream = Readable.from([encoded.text]) + return TextStream.withFallback(filepath, (stream) => readLines(stream, opts)) +} + +async function readLines(stream: Readable, opts: { limit: number; offset: number }) { // kilocode_change end const rl = createInterface({ input: stream, From 67e9fdc2cf8c9704229fc9f1ecbbdfd91778d940 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Fri, 8 May 2026 19:33:11 +0000 Subject: [PATCH 2/2] fix(cli): destroy raw fs read stream on consumer teardown When the consumer (readLines) hits the line/byte cap and destroys the PassThrough, the underlying createReadStream had no link back and would keep reading chunks to EOF in the background, defeating the early-exit optimisation for large files. --- packages/opencode/src/kilocode/text-stream.ts | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/packages/opencode/src/kilocode/text-stream.ts b/packages/opencode/src/kilocode/text-stream.ts index ae2d3de6db9..1bb4d55f50e 100644 --- a/packages/opencode/src/kilocode/text-stream.ts +++ b/packages/opencode/src/kilocode/text-stream.ts @@ -4,22 +4,13 @@ import * as Encoding from "./encoding" /** * Encoding-aware text streaming for tools that walk a file line by line. + * Optimistically stream as UTF-8; fall back to a buffered iconv decode only + * when the bytes turn out not to be valid UTF-8. * - * Most files we read are UTF-8, so we stream chunks straight from disk - * (decoded strictly with `TextDecoder({ fatal: true })`) and let the consumer - * early-exit without buffering the rest of the file. Only when the bytes turn - * out not to be valid UTF-8 do we fall back to {@link Encoding.read}, which - * runs full-file detection through iconv-lite. - * - * Consumers should import this module as a namespace: * import * as TextStream from "../kilocode/text-stream" */ -/** - * Sentinel error used to signal the optimistic UTF-8 stream gave up because - * the bytes are not valid UTF-8. Distinct class so {@link withFallback} can - * tell it apart from real I/O failures. - */ +/** Distinct class so {@link withFallback} can tell us apart from real I/O failures. */ export class InvalidUtf8Error extends Error { constructor() { super("invalid utf-8") @@ -27,11 +18,8 @@ export class InvalidUtf8Error extends Error { } /** - * UTF-8 text Readable for `filepath`. Streams chunks straight from disk so the - * caller can early-exit without buffering the rest of the file. If invalid - * UTF-8 is encountered the stream is destroyed with an {@link - * InvalidUtf8Error}. A leading UTF-8 BOM passes through as U+FEFF — the same - * behaviour as `createReadStream({ encoding: "utf8" })`. + * UTF-8 text Readable for `filepath`. A leading UTF-8 BOM passes through as + * U+FEFF — same as `createReadStream({ encoding: "utf8" })`. */ export function openUtf8(filepath: string): Readable { const out = new PassThrough({ encoding: "utf8" }) @@ -56,23 +44,21 @@ export function openUtf8(filepath: string): Readable { } }) raw.on("error", (err) => out.destroy(err)) + // Propagate consumer-side teardown so early-exit (line / byte cap, fallback) + // stops pulling chunks from disk instead of running to EOF. + out.on("close", () => raw.destroy()) return out } -/** - * Whole-file UTF-8 text Readable for `filepath`, decoded via - * {@link Encoding.read}'s detection logic. Buffers the entire decoded file - * into memory; used as a fallback for files that {@link openUtf8} rejects. - */ +/** Whole-file UTF-8 Readable via {@link Encoding.read}; buffers the entire decoded file. */ export async function openDecoded(filepath: string): Promise { const decoded = await Encoding.read(filepath) return Readable.from([decoded.text]) } /** - * Run `fn` against an optimistic UTF-8 stream of `filepath`. If the bytes - * turn out not to be valid UTF-8, `fn` is run a second time against a - * fallback iconv-decoded stream. Other errors are propagated unchanged. + * Run `fn` against an optimistic UTF-8 stream; on {@link InvalidUtf8Error} + * retry once against {@link openDecoded}. Other errors propagate. */ export async function withFallback(filepath: string, fn: (input: Readable) => Promise): Promise { try {