diff --git a/src/bun_core/string/immutable/unicode.rs b/src/bun_core/string/immutable/unicode.rs index a15a452ec216..3326df6c9049 100644 --- a/src/bun_core/string/immutable/unicode.rs +++ b/src/bun_core/string/immutable/unicode.rs @@ -146,16 +146,11 @@ pub fn to_utf8_list_with_type_bun( let count: usize = replacement.utf8_width() as usize; #[cfg(not(target_family = "wasm"))] - { - let extra = ((utf16_remaining.len() as u64 & ((1u64 << 52) - 1)) as f64 * 1.2) as usize; - list.reserve_exact((i + count + list.len() + extra).saturating_sub(list.len())); - } + let extra = ((utf16_remaining.len() as u64 & ((1u64 << 52) - 1)) as f64 * 1.2) as usize; #[cfg(target_family = "wasm")] - { - list.reserve_exact( - (i + count + list.len() + utf16_remaining.len() + 4).saturating_sub(list.len()), - ); - } + let extra = utf16_remaining.len() + 4; + list.try_reserve_exact(i + count + extra) + .map_err(|_| AllocError)?; append_u16_as_u8(list, to_copy); if SKIP_TRAILING_REPLACEMENT { @@ -174,8 +169,8 @@ pub fn to_utf8_list_with_type_bun( } if !utf16_remaining.is_empty() { - let need = utf16_remaining.len() + list.len(); - list.reserve_exact(need.saturating_sub(list.len())); + list.try_reserve_exact(utf16_remaining.len()) + .map_err(|_| AllocError)?; append_u16_as_u8(list, utf16_remaining); } diff --git a/src/runtime/webcore/TextEncoderStreamEncoder.rs b/src/runtime/webcore/TextEncoderStreamEncoder.rs index eb8dc576846e..10902475b363 100644 --- a/src/runtime/webcore/TextEncoderStreamEncoder.rs +++ b/src/runtime/webcore/TextEncoderStreamEncoder.rs @@ -1,7 +1,7 @@ use core::cell::Cell; use core::ptr::NonNull; -use bun_collections::VecExt as _; +use bun_alloc::AllocError; use bun_core::strings; use bun_jsc::{JSGlobalObject, JSUint8Array, JSValue}; use bun_ptr::RawSlice; @@ -30,18 +30,20 @@ impl TextEncoderStreamEncoder { return JSUint8Array::create_empty(global); } let mut buffer = Vec::new(); - self.encode_latin1_into(input, &mut buffer); + if self.encode_latin1_into(input, &mut buffer).is_err() { + return global.throw_out_of_memory_value(); + } JSUint8Array::from_bytes(global, buffer.into()) } - fn encode_latin1_into(&self, input: &[u8], buffer: &mut Vec) { + fn encode_latin1_into(&self, input: &[u8], buffer: &mut Vec) -> Result<(), AllocError> { bun_output::scoped_log!( TextEncoderStreamEncoder, "encodeLatin1: \"{}\"", bstr::BStr::new(input) ); if input.is_empty() { - return; + return Ok(()); } let prepend_replacement_len: usize = if self.pending_lead_surrogate.take().is_some() { @@ -57,7 +59,9 @@ impl TextEncoderStreamEncoder { // 278.00 ms 13.0% 278.00 ms simdutf::arm64::implementation::utf8_length_from_latin1(char const*, unsigned long) const // // - buffer.reserve(input.len() + prepend_replacement_len); + buffer + .try_reserve(input.len() + prepend_replacement_len) + .map_err(|_| AllocError)?; if prepend_replacement_len > 0 { buffer.extend_from_slice(&[0xef, 0xbf, 0xbd]); } @@ -75,14 +79,17 @@ impl TextEncoderStreamEncoder { remain = &remain[result.read as usize..]; if result.written == 0 && result.read == 0 { - buffer.reserve(2); + buffer.try_reserve(2).map_err(|_| AllocError)?; } else if buffer.len() == buffer.capacity() && !remain.is_empty() { - buffer.ensure_total_capacity(buffer.len() + remain.len() + 1); + buffer + .try_reserve(remain.len() + 1) + .map_err(|_| AllocError)?; } } debug_assert!( buffer.len() == (simdutf::length::utf8::from::latin1(input) + prepend_replacement_len) ); + Ok(()) } fn encode_utf16(&self, global: &JSGlobalObject, input: &[u16]) -> JSValue { @@ -99,7 +106,7 @@ impl TextEncoderStreamEncoder { JSUint8Array::from_bytes(global, buf.into()) } - fn encode_utf16_into(&self, input: &[u16], buf: &mut Vec) -> Result<(), ()> { + fn encode_utf16_into(&self, input: &[u16], buf: &mut Vec) -> Result<(), AllocError> { bun_output::scoped_log!( TextEncoderStreamEncoder, "encodeUTF16: \"{}\"", @@ -144,7 +151,9 @@ impl TextEncoderStreamEncoder { remain = &remain[1..]; if remain.is_empty() { - buf.extend_from_slice(&sequence[0..converted.utf8_width() as usize]); + let width = converted.utf8_width() as usize; + buf.try_reserve(width).map_err(|_| AllocError)?; + buf.extend_from_slice(&sequence[0..width]); return Ok(()); } @@ -158,13 +167,14 @@ impl TextEncoderStreamEncoder { let length = simdutf::length::utf8::from::utf16::le(remain); - buf.reserve( + buf.try_reserve( length + match prepend { Some(pre) => pre.len as usize, None => 0, }, - ); + ) + .map_err(|_| AllocError)?; if let Some(pre) = &prepend { buf.extend_from_slice(&pre.bytes[0..pre.len as usize]); @@ -188,8 +198,7 @@ impl TextEncoderStreamEncoder { if result.status != simdutf::Status::SUCCESS { // Slow path: there was invalid UTF-16, so we need to convert it without simdutf. - let lead_surrogate = - strings::to_utf8_list_with_type_bun::(buf, remain).map_err(|_| ())?; + let lead_surrogate = strings::to_utf8_list_with_type_bun::(buf, remain)?; if let Some(pending_lead) = lead_surrogate { self.pending_lead_surrogate.set(Some(pending_lead)); } @@ -289,15 +298,13 @@ pub extern "C" fn TextEncoderStreamEncoder__encodeIntoSink( // (theoretical) re-entrant encode-into-sink call cannot BorrowMut-panic. let mut buf = this.scratch.take(); buf.clear(); - if str.is_16bit() { - if this - .encode_utf16_into(str.utf16_slice_aligned(), &mut buf) - .is_err() - { - return global.throw_out_of_memory_value(); - } + let encoded = if str.is_16bit() { + this.encode_utf16_into(str.utf16_slice_aligned(), &mut buf) } else { - this.encode_latin1_into(str.slice(), &mut buf); + this.encode_latin1_into(str.slice(), &mut buf) + }; + if encoded.is_err() { + return global.throw_out_of_memory_value(); } if buf.is_empty() { this.scratch.replace(buf); diff --git a/test/js/web/encoding/text-encoder-stream.test.ts b/test/js/web/encoding/text-encoder-stream.test.ts index 5c2578ec9479..d37ff1663d23 100644 --- a/test/js/web/encoding/text-encoder-stream.test.ts +++ b/test/js/web/encoding/text-encoder-stream.test.ts @@ -1,5 +1,5 @@ -import { expect, test } from "bun:test"; -import { readableStreamFromArray } from "harness"; +import { describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, isASAN, readableStreamFromArray } from "harness"; // META: global=window,worker // META: script=resources/readable-stream-from-array.js @@ -267,3 +267,162 @@ test("TextEncoderStream -> TextDecoderStream -> TextEncoderStream -> native HTTP expect(out.byteLength).toBe(expected.byteLength); expect(Buffer.compare(out, Buffer.from(expected))).toBe(0); }); + +// The encoder sizes a chunk's output buffer from the chunk itself (up to three +// bytes per UTF-16 unit), so those reservations are where a TextEncoderStream +// runs out of memory. A failed one has to error the stream like any other +// transform failure, not abort the process (#33014 fixed the decoding direction +// the same way). ASAN's per-allocation cap makes the failure deterministic: the +// input strings are allocated by JSC and are not subject to it, while every +// encoder reservation above CAP_MB fails. Each input is shaped so that a +// different reservation in the encoder is the one that fails. (Bun's own startup +// needs allocations of up to about 1.5 MiB, so the cap cannot go much lower; the +// Latin-1 growth cases encode three quarters of it through the debug build's +// per-char loop, so it should not be much higher either.) +describe.skipIf(!isASAN)("a failed output buffer allocation errors the stream instead of aborting", () => { + const CAP_MB = 4; + const env = { + ...bunEnv, + // detect_leaks=0: natives owned only by a JSC cell are invisible to + // LeakSanitizer's reachability scan and would be reported at exit. + ASAN_OPTIONS: [ + bunEnv.ASAN_OPTIONS, + "allocator_may_return_null=1", + `max_allocation_size_mb=${CAP_MB}`, + "detect_leaks=0", + ] + .filter(Boolean) + .join(":"), + }; + const prelude = /* js */ ` + const MiB = 1024 * 1024; + const inputs = { + // Latin-1: the up-front reservation (one byte per input byte) is above the cap. + latin1: () => Buffer.alloc(${2 * CAP_MB} * MiB, "a").toString("latin1"), + // Latin-1: the up-front reservation (3/4 of the cap) fits, but every byte + // encodes to two, so the buffer fills up half way through and the reservation + // growing it (to at least 1.5x the input) fails. + latin1Grow: () => Buffer.alloc(${0.75 * CAP_MB} * MiB, 0xe9).toString("latin1"), + // Latin-1: an ASCII byte followed by an odd number of two-byte chars leaves a + // single spare byte, so a pass encodes nothing and the branch reserving room + // for one more char is the one that fails. + latin1Stuck: () => { + const bytes = Buffer.alloc(${0.75 * CAP_MB} * MiB + 2, 0xe9); + bytes[0] = 0x61; + return bytes.toString("latin1"); + }, + // UTF-16 fast path: simdutf predicts three bytes per unit, 1.5x the cap. + utf16: () => Buffer.alloc(${CAP_MB} * MiB, "\\u65e5", "utf16le").toString("utf16le"), + // UTF-16 slow path, N ASCII units behind a lone surrogate (the surrogate + // makes the concatenation a 16-bit string): simdutf's prediction (N + 2 + // bytes) is reserved fine, then the surrogate hands the chunk to the + // replacement encoder, whose first reservation (1.2 bytes per remaining + // unit, 1.2N + 3) does not fit. + utf16Invalid: () => "\\ud800" + Buffer.alloc(${CAP_MB - 0.25} * MiB, "a").toString("latin1"), + }; + const describeError = e => ({ name: e.name, message: e.message }); + const SMALL = "ok\\u00e9"; + const LEADING = ${JSON.stringify(leading)}; + `; + const outOfMemory = { name: "RangeError", message: "Out of memory" }; + const smallEncoded = [0x6f, 0x6b, 0xc3, 0xa9]; + + async function runChild(script: string) { + await using proc = Bun.spawn({ + cmd: [bunExe(), "-e", prelude + script], + env, + stdout: "pipe", + // ASAN prints a "failed to allocate" warning for every refused allocation; + // drain it, don't assert on it. + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + const result = JSON.parse(stdout.trim() || JSON.stringify({ stdout, stderr, exitCode })); + return { result, exitCode }; + } + + test.concurrent("output enqueued to JS: the write rejects and the readable errors", async () => { + const { result, exitCode } = await runChild(/* js */ ` + const results = {}; + for (const [name, makeInput] of Object.entries(inputs)) { + const stream = new TextEncoderStream(); + const reader = stream.readable.getReader(); + const writer = stream.writable.getWriter(); + const read = reader.read(); + results[name] = { + write: await writer.write(makeInput()).then(() => "resolved", describeError), + read: await read.then(chunk => "resolved with " + chunk.value?.length + " bytes", describeError), + }; + } + // Encoding still works in this process afterwards. + const stream = new TextEncoderStream(); + const writer = stream.writable.getWriter(); + writer.write(SMALL); + writer.close(); + results.afterwards = Array.from(await new Response(stream.readable).bytes()); + console.log(JSON.stringify(results)); + `); + expect(result).toEqual({ + latin1: { write: outOfMemory, read: outOfMemory }, + latin1Grow: { write: outOfMemory, read: outOfMemory }, + latin1Stuck: { write: outOfMemory, read: outOfMemory }, + utf16: { write: outOfMemory, read: outOfMemory }, + utf16Invalid: { write: outOfMemory, read: outOfMemory }, + afterwards: smallEncoded, + }); + expect(exitCode).toBe(0); + }); + + // With a native sink attached (Bun.serve's response sink here) the encoder + // writes into its own buffer and hands that to the sink instead of enqueueing + // Uint8Arrays: a separate entry point into the same encoders. Each chunk is + // preceded by a dangling lead surrogate, so the output has to be assembled in + // the encoder's buffer (the replacement goes in front of the chunk's bytes); + // that is the case the sink path owns even if plain chunks are ever handed to + // the sink directly (#36877), and it also covers the prepend variant of every + // reservation. The errored transform cancels the stream piped into it, so the + // source's cancel reason is where the error shows up. + test.concurrent("output written to a native sink: the transform errors and the server keeps serving", async () => { + const { result, exitCode } = await runChild(/* js */ ` + const cancelReasons = {}; + const server = Bun.serve({ + port: 0, + fetch(req) { + const name = new URL(req.url).pathname.slice(1); + const { promise, resolve } = Promise.withResolvers(); + cancelReasons[name] = promise; + const source = new ReadableStream({ + start(controller) { + controller.enqueue(LEADING); + if (name in inputs) { + controller.enqueue(inputs[name]()); + } else { + controller.enqueue(SMALL); + controller.close(); + } + }, + cancel: reason => resolve(describeError(reason)), + }); + return new Response(source.pipeThrough(new TextEncoderStream())); + }, + }); + const results = {}; + for (const name of Object.keys(inputs)) { + await fetch(new URL(name, server.url)).then(response => response.arrayBuffer(), () => {}); + results[name] = await cancelReasons[name]; + } + results.afterwards = Array.from(await (await fetch(new URL("small", server.url))).bytes()); + server.stop(true); + console.log(JSON.stringify(results)); + `); + expect(result).toEqual({ + latin1: outOfMemory, + latin1Grow: outOfMemory, + latin1Stuck: outOfMemory, + utf16: outOfMemory, + utf16Invalid: outOfMemory, + afterwards: replacementEncoded.concat(smallEncoded), + }); + expect(exitCode).toBe(0); + }); +});