diff --git a/scripts/verify-baseline-static/allowlist-aarch64.txt b/scripts/verify-baseline-static/allowlist-aarch64.txt index ba6cfcf7489e..d0faafd7b4d9 100644 --- a/scripts/verify-baseline-static/allowlist-aarch64.txt +++ b/scripts/verify-baseline-static/allowlist-aarch64.txt @@ -5,7 +5,7 @@ # ---------------------------------------------------------------------------- # Bun's Highway SVE/SVE2 targets. Gate: hwy::SupportedTargets via getauxval(AT_HWCAP). -# (80 symbols) +# (84 symbols) # ---------------------------------------------------------------------------- _ZN3bun10N_SVE2_12810MemMemImplEPKhmS2_m [SVE] _ZN3bun10N_SVE2_12815CopyU16ToU8ImplEPKtmPh [SVE] @@ -26,6 +26,7 @@ _ZN3bun10N_SVE2_12836ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [S _ZN3bun10N_SVE2_12838IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [SVE] _ZN3bun10N_SVE2_12846IndexOfInterestingCharacterInStringLiteralImplEPKhmh [SVE] _ZN3bun10N_SVE2_12846IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [SVE] +_ZN3bun10N_SVE2_12849IndexOfInterestingCharacterInMultilineCommentImplEPKhm [SVE] _ZN3bun10N_SVE2_12849IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [SVE] _ZN3bun5N_SVE10MemMemImplEPKhmS2_m [SVE] _ZN3bun5N_SVE15CopyU16ToU8ImplEPKtmPh [SVE] @@ -46,6 +47,7 @@ _ZN3bun5N_SVE36ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [S _ZN3bun5N_SVE38IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [SVE] _ZN3bun5N_SVE46IndexOfInterestingCharacterInStringLiteralImplEPKhmh [SVE] _ZN3bun5N_SVE46IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [SVE] +_ZN3bun5N_SVE49IndexOfInterestingCharacterInMultilineCommentImplEPKhm [SVE] _ZN3bun5N_SVE49IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [SVE] _ZN3bun6N_SVE210MemMemImplEPKhmS2_m [SVE] _ZN3bun6N_SVE215CopyU16ToU8ImplEPKtmPh [SVE] @@ -66,6 +68,7 @@ _ZN3bun6N_SVE236ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [S _ZN3bun6N_SVE238IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [SVE] _ZN3bun6N_SVE246IndexOfInterestingCharacterInStringLiteralImplEPKhmh [SVE] _ZN3bun6N_SVE246IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [SVE] +_ZN3bun6N_SVE249IndexOfInterestingCharacterInMultilineCommentImplEPKhm [SVE] _ZN3bun6N_SVE249IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [SVE] _ZN3bun9N_SVE_25610MemMemImplEPKhmS2_m [SVE] _ZN3bun9N_SVE_25615CopyU16ToU8ImplEPKtmPh [SVE] @@ -86,6 +89,7 @@ _ZN3bun9N_SVE_25636ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [S _ZN3bun9N_SVE_25638IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [SVE] _ZN3bun9N_SVE_25646IndexOfInterestingCharacterInStringLiteralImplEPKhmh [SVE] _ZN3bun9N_SVE_25646IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [SVE] +_ZN3bun9N_SVE_25649IndexOfInterestingCharacterInMultilineCommentImplEPKhm [SVE] _ZN3bun9N_SVE_25649IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [SVE] diff --git a/scripts/verify-baseline-static/allowlist-x64-windows.txt b/scripts/verify-baseline-static/allowlist-x64-windows.txt index 21a06ef1a2de..7daa86ff3fcb 100644 --- a/scripts/verify-baseline-static/allowlist-x64-windows.txt +++ b/scripts/verify-baseline-static/allowlist-x64-windows.txt @@ -437,7 +437,7 @@ ctiMasmProbeTrampolineAVX [AVX] # ---------------------------------------------------------------------------- # Highway. MSVC-mangled bun::N_AVX* names. -# (120 symbols) +# (126 symbols) # ---------------------------------------------------------------------------- bun::N_AVX10_2::ContainsNewlineOrNonASCIIOrQuoteImpl [AVX, AVX512BW, AVX512F] bun::N_AVX10_2::CopyAsciiPrefixImpl [AVX, AVX512BW, AVX512F, AVX512VL] @@ -449,6 +449,7 @@ bun::N_AVX10_2::FirstNonAscii16Impl [AVX, AVX51 bun::N_AVX10_2::FirstNonAscii8Impl [AVX, AVX512BW] bun::N_AVX10_2::IndexOfAnyCharImpl [AVX, AVX512BW, AVX512F, AVX512VL, AVX512_FP16] bun::N_AVX10_2::IndexOfCharImpl [AVX, AVX512BW, BMI2] +bun::N_AVX10_2::IndexOfInterestingCharacterInMultilineCommentImpl [AVX, AVX512BW, AVX512F] bun::N_AVX10_2::IndexOfInterestingCharacterInStringLiteralImpl [AVX, AVX512BW, AVX512F] bun::N_AVX10_2::IndexOfNeedsEscapeForJavaScriptStringImplBacktick [AVX, AVX512BW, AVX512F] bun::N_AVX10_2::IndexOfNeedsEscapeForJavaScriptStringImplQuote [AVX, AVX512BW, AVX512F] @@ -469,6 +470,7 @@ bun::N_AVX2::FirstNonAscii16Impl [AVX, AVX2, bun::N_AVX2::FirstNonAscii8Impl [AVX, AVX2] bun::N_AVX2::IndexOfAnyCharImpl [AVX, AVX2] bun::N_AVX2::IndexOfCharImpl [AVX, AVX2] +bun::N_AVX2::IndexOfInterestingCharacterInMultilineCommentImpl [AVX, AVX2] bun::N_AVX2::IndexOfInterestingCharacterInStringLiteralImpl [AVX, AVX2] bun::N_AVX2::IndexOfNeedsEscapeForJavaScriptStringImplBacktick [AVX, AVX2] bun::N_AVX2::IndexOfNeedsEscapeForJavaScriptStringImplQuote [AVX, AVX2] @@ -489,6 +491,7 @@ bun::N_AVX3::FirstNonAscii16Impl [AVX, AVX51 bun::N_AVX3::FirstNonAscii8Impl [AVX, AVX512BW] bun::N_AVX3::IndexOfAnyCharImpl [AVX, AVX512BW, AVX512F, AVX512VL] bun::N_AVX3::IndexOfCharImpl [AVX, AVX512BW, BMI2] +bun::N_AVX3::IndexOfInterestingCharacterInMultilineCommentImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3::IndexOfInterestingCharacterInStringLiteralImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3::IndexOfNeedsEscapeForJavaScriptStringImplBacktick [AVX, AVX512BW, AVX512F] bun::N_AVX3::IndexOfNeedsEscapeForJavaScriptStringImplQuote [AVX, AVX512BW, AVX512F] @@ -509,6 +512,7 @@ bun::N_AVX3_DL::FirstNonAscii16Impl [AVX, AVX51 bun::N_AVX3_DL::FirstNonAscii8Impl [AVX, AVX512BW] bun::N_AVX3_DL::IndexOfAnyCharImpl [AVX, AVX512BW, AVX512F, AVX512VL] bun::N_AVX3_DL::IndexOfCharImpl [AVX, AVX512BW, BMI2] +bun::N_AVX3_DL::IndexOfInterestingCharacterInMultilineCommentImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3_DL::IndexOfInterestingCharacterInStringLiteralImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3_DL::IndexOfNeedsEscapeForJavaScriptStringImplBacktick [AVX, AVX512BW, AVX512F] bun::N_AVX3_DL::IndexOfNeedsEscapeForJavaScriptStringImplQuote [AVX, AVX512BW, AVX512F] @@ -529,6 +533,7 @@ bun::N_AVX3_SPR::FirstNonAscii16Impl [AVX, AVX51 bun::N_AVX3_SPR::FirstNonAscii8Impl [AVX, AVX512BW] bun::N_AVX3_SPR::IndexOfAnyCharImpl [AVX, AVX512BW, AVX512F, AVX512VL, AVX512_FP16] bun::N_AVX3_SPR::IndexOfCharImpl [AVX, AVX512BW, BMI2] +bun::N_AVX3_SPR::IndexOfInterestingCharacterInMultilineCommentImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3_SPR::IndexOfInterestingCharacterInStringLiteralImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3_SPR::IndexOfNeedsEscapeForJavaScriptStringImplBacktick [AVX, AVX512BW, AVX512F] bun::N_AVX3_SPR::IndexOfNeedsEscapeForJavaScriptStringImplQuote [AVX, AVX512BW, AVX512F] @@ -549,6 +554,7 @@ bun::N_AVX3_ZEN4::FirstNonAscii16Impl [AVX, AVX51 bun::N_AVX3_ZEN4::FirstNonAscii8Impl [AVX, AVX512BW] bun::N_AVX3_ZEN4::IndexOfAnyCharImpl [AVX, AVX512BW, AVX512F, AVX512VL] bun::N_AVX3_ZEN4::IndexOfCharImpl [AVX, AVX512BW, BMI2] +bun::N_AVX3_ZEN4::IndexOfInterestingCharacterInMultilineCommentImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3_ZEN4::IndexOfInterestingCharacterInStringLiteralImpl [AVX, AVX512BW, AVX512F] bun::N_AVX3_ZEN4::IndexOfNeedsEscapeForJavaScriptStringImplBacktick [AVX, AVX512BW, AVX512F] bun::N_AVX3_ZEN4::IndexOfNeedsEscapeForJavaScriptStringImplQuote [AVX, AVX512BW, AVX512F] diff --git a/scripts/verify-baseline-static/allowlist-x64.txt b/scripts/verify-baseline-static/allowlist-x64.txt index fef858203c71..4ca0b310dd8a 100644 --- a/scripts/verify-baseline-static/allowlist-x64.txt +++ b/scripts/verify-baseline-static/allowlist-x64.txt @@ -459,7 +459,7 @@ ctiMasmProbeTrampolineAVX [AVX] # ---------------------------------------------------------------------------- # Bun's Highway SIMD. Gate: HWY_DYNAMIC_DISPATCH via hwy::SupportedTargets. -# (120 symbols) +# (126 symbols) # ---------------------------------------------------------------------------- _ZN3bun10N_AVX3_SPR10MemMemImplEPKhmS2_m [AVX, AVX512BW, AVX512F, BMI1] _ZN3bun10N_AVX3_SPR15CopyU16ToU8ImplEPKtmPh [AVX, AVX512BW, AVX512F, AVX512VL, AVX512_VBMI] @@ -480,6 +480,7 @@ _ZN3bun10N_AVX3_SPR36ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [ _ZN3bun10N_AVX3_SPR38IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun10N_AVX3_SPR46IndexOfInterestingCharacterInStringLiteralImplEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun10N_AVX3_SPR46IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [AVX, AVX512BW, AVX512F] +_ZN3bun10N_AVX3_SPR49IndexOfInterestingCharacterInMultilineCommentImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun10N_AVX3_SPR49IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun11N_AVX3_ZEN410MemMemImplEPKhmS2_m [AVX, AVX512BW, AVX512F, BMI1] _ZN3bun11N_AVX3_ZEN415CopyU16ToU8ImplEPKtmPh [AVX, AVX512BW, AVX512F, AVX512VL, AVX512_VBMI] @@ -500,6 +501,7 @@ _ZN3bun11N_AVX3_ZEN436ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [ _ZN3bun11N_AVX3_ZEN438IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun11N_AVX3_ZEN446IndexOfInterestingCharacterInStringLiteralImplEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun11N_AVX3_ZEN446IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [AVX, AVX512BW, AVX512F] +_ZN3bun11N_AVX3_ZEN449IndexOfInterestingCharacterInMultilineCommentImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun11N_AVX3_ZEN449IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun6N_AVX210MemMemImplEPKhmS2_m [AVX, AVX2] _ZN3bun6N_AVX215CopyU16ToU8ImplEPKtmPh [AVX, AVX2] @@ -520,6 +522,7 @@ _ZN3bun6N_AVX236ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [ _ZN3bun6N_AVX238IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [AVX, AVX2] _ZN3bun6N_AVX246IndexOfInterestingCharacterInStringLiteralImplEPKhmh [AVX, AVX2] _ZN3bun6N_AVX246IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [AVX, AVX2] +_ZN3bun6N_AVX249IndexOfInterestingCharacterInMultilineCommentImplEPKhm [AVX, AVX2] _ZN3bun6N_AVX249IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [AVX, AVX2] _ZN3bun6N_AVX310MemMemImplEPKhmS2_m [AVX, AVX512BW, AVX512F, BMI1] _ZN3bun6N_AVX315CopyU16ToU8ImplEPKtmPh [AVX, AVX512BW, AVX512F, AVX512VL] @@ -540,6 +543,7 @@ _ZN3bun6N_AVX336ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [ _ZN3bun6N_AVX338IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun6N_AVX346IndexOfInterestingCharacterInStringLiteralImplEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun6N_AVX346IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [AVX, AVX512BW, AVX512F] +_ZN3bun6N_AVX349IndexOfInterestingCharacterInMultilineCommentImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun6N_AVX349IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX10_210MemMemImplEPKhmS2_m [AVX, AVX512BW, AVX512F, BMI1] _ZN3bun9N_AVX10_215CopyU16ToU8ImplEPKtmPh [AVX, AVX512BW, AVX512F, AVX512VL, AVX512_VBMI] @@ -560,6 +564,7 @@ _ZN3bun9N_AVX10_236ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [ _ZN3bun9N_AVX10_238IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX10_246IndexOfInterestingCharacterInStringLiteralImplEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX10_246IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [AVX, AVX512BW, AVX512F] +_ZN3bun9N_AVX10_249IndexOfInterestingCharacterInMultilineCommentImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX10_249IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX3_DL10MemMemImplEPKhmS2_m [AVX, AVX512BW, AVX512F, BMI1] _ZN3bun9N_AVX3_DL15CopyU16ToU8ImplEPKtmPh [AVX, AVX512BW, AVX512F, AVX512VL, AVX512_VBMI] @@ -580,6 +585,7 @@ _ZN3bun9N_AVX3_DL36ContainsNewlineOrNonASCIIOrQuoteImplEPKhm [ _ZN3bun9N_AVX3_DL38IndexOfNewlineOrNonASCIIOrHashOrAtImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX3_DL46IndexOfInterestingCharacterInStringLiteralImplEPKhmh [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX3_DL46IndexOfNeedsEscapeForJavaScriptStringImplQuoteEPKhmh [AVX, AVX512BW, AVX512F] +_ZN3bun9N_AVX3_DL49IndexOfInterestingCharacterInMultilineCommentImplEPKhm [AVX, AVX512BW, AVX512F] _ZN3bun9N_AVX3_DL49IndexOfNeedsEscapeForJavaScriptStringImplBacktickEPKhmh [AVX, AVX512BW, AVX512F] diff --git a/src/highway/lib.rs b/src/highway/lib.rs index 240c14b846f7..fb66cb27da57 100644 --- a/src/highway/lib.rs +++ b/src/highway/lib.rs @@ -12,6 +12,11 @@ unsafe extern "C" { quote: u8, ) -> usize; + fn highway_index_of_interesting_character_in_multiline_comment( + text: *const u8, + text_len: usize, + ) -> usize; + fn highway_index_of_newline_or_non_ascii(haystack: *const u8, haystack_len: usize) -> usize; fn highway_index_of_newline_or_non_ascii_or_hash_or_at( @@ -121,6 +126,41 @@ pub fn index_of_interesting_character_in_string_literal( Some(result) } +/// Useful for scanning the body of `/* ... */` block comments. +/// Scans for: +/// - `*` (potential `*/` terminator) +/// - `\n`, `\r` +/// - Non-ASCII characters (so the caller decodes U+2028/U+2029 and other +/// multi-byte sequences one code point at a time) +#[inline(always)] +pub fn index_of_interesting_character_in_multiline_comment(slice: &[u8]) -> Option { + if slice.is_empty() { + return None; + } + + // SAFETY: slice.ptr/len are a valid readable range. + let result = unsafe { + highway_index_of_interesting_character_in_multiline_comment(slice.as_ptr(), slice.len()) + }; + + if result == slice.len() { + return None; + } + + if cfg!(debug_assertions) { + let haystack_char = slice[result]; + if !(haystack_char > 127 + || haystack_char == b'*' + || haystack_char == b'\r' + || haystack_char == b'\n') + { + panic!("Invalid character found in indexOfInterestingCharacterInMultilineComment"); + } + } + + Some(result) +} + #[inline(always)] pub fn index_of_newline_or_non_ascii(haystack: &[u8]) -> Option { debug_assert!(!haystack.is_empty()); diff --git a/src/js_parser/lexer.rs b/src/js_parser/lexer.rs index a6119267fe28..3c08e85c22b7 100644 --- a/src/js_parser/lexer.rs +++ b/src/js_parser/lexer.rs @@ -2441,25 +2441,14 @@ lexer_impl_header! { )?; } _ => { - if Environment::ENABLE_SIMD { - if self.code_point < 128 { - let remainder = &contents[self.current..]; - if remainder.len() >= 512 { - match skip_to_interesting_character_in_multiline_comment( - remainder, - ) { - Some(off) => { - self.current += off as usize; - self.end = self.current.saturating_sub(1); - self.step_with(contents); - continue; - } - None => { - self.step_with(contents); - continue; - } - } - } + if self.code_point < 128 { + let remainder = &contents[self.current..]; + if remainder.len() >= 512 { + self.current += + skip_to_interesting_character_in_multiline_comment(remainder); + self.end = self.current.saturating_sub(1); + self.step_with(contents); + continue; } } @@ -4158,29 +4147,14 @@ impl PragmaArg { } } -fn skip_to_interesting_character_in_multiline_comment(text_: &[u8]) -> Option { - // PERF(port): Zig uses portable @Vector SIMD here. Rust port uses scalar; could - // swap to bun_highway or core::simd. Logic preserved (returns offset of first - // '*' / '\r' / '\n' / non-ASCII byte, truncated to chunks of `ascii_vector_size`). - // TODO(port): SIMD reimplementation - let vsize = strings::ASCII_VECTOR_SIZE; - let text_end_len = text_.len() & !(vsize - 1); - debug_assert!(text_end_len.is_multiple_of(vsize)); - debug_assert!(text_end_len <= text_.len()); - - let mut off: usize = 0; - while off < text_end_len { - let chunk = &text_[off..off + vsize]; - for (j, &b) in chunk.iter().enumerate() { - if b > 127 || b == b'*' || b == b'\r' || b == b'\n' { - debug_assert!(j < vsize); - return Some((off + j) as u32); - } - } - off += vsize; - } - - Some(off as u32) +/// Byte offset of the next character `scan_multi_line_comment_body` has to +/// inspect one code point at a time: the first `*` (potential `*/` +/// terminator), `\r` / `\n` (newline tracking for ASI), or non-ASCII byte +/// (U+2028/U+2029 and other multi-byte sequences). Returns `text_.len()` when +/// the rest of the input has no such byte — the comment is unterminated, so +/// the caller's next `step()` lands on EOF and reports the error. +fn skip_to_interesting_character_in_multiline_comment(text_: &[u8]) -> usize { + bun_highway::index_of_interesting_character_in_multiline_comment(text_).unwrap_or(text_.len()) } fn index_of_interesting_character_in_string_literal(text_: &[u8], quote: u8) -> Option { diff --git a/src/jsc/bindings/highway_strings.cpp b/src/jsc/bindings/highway_strings.cpp index ea52a48e17b9..27d6766b9dbd 100644 --- a/src/jsc/bindings/highway_strings.cpp +++ b/src/jsc/bindings/highway_strings.cpp @@ -287,6 +287,50 @@ size_t IndexOfInterestingCharacterInStringLiteralImpl(const uint8_t* HWY_RESTRIC return text_len; } +// Scans the body of a `/* ... */` block comment for the next byte the lexer +// must inspect one code point at a time: `*` (potential `*/` terminator), +// `\r` / `\n` (newline tracking for ASI), or any non-ASCII byte (so U+2028 / +// U+2029 and other multi-byte sequences are decoded by the scalar path). +size_t IndexOfInterestingCharacterInMultilineCommentImpl(const uint8_t* HWY_RESTRICT text, size_t text_len) +{ + ASSERT(text_len > 0); + + D8 d; + const size_t N = hn::Lanes(d); + + const auto vec_star = hn::Set(d, '*'); + const auto vec_carriage = hn::Set(d, '\r'); + const auto vec_newline = hn::Set(d, '\n'); + const auto vec_max_ascii = hn::Set(d, uint8_t { 127 }); + + size_t i = 0; + const size_t simd_text_len = text_len - (text_len % N); + for (; i < simd_text_len; i += N) { + const auto vec = hn::LoadU(d, text + i); + + const auto mask_star = hn::Eq(vec, vec_star); + const auto mask_carriage = hn::Eq(vec, vec_carriage); + const auto mask_newline = hn::Eq(vec, vec_newline); + const auto mask_non_ascii = hn::Gt(vec, vec_max_ascii); + + const auto found_mask = hn::Or(hn::Or(mask_star, mask_non_ascii), hn::Or(mask_carriage, mask_newline)); + + const intptr_t pos = hn::FindFirstTrue(d, found_mask); + if (pos >= 0) { + return i + pos; + } + } + + for (; i < text_len; ++i) { + const uint8_t char_ = text[i]; + if (char_ == '*' || char_ == '\r' || char_ == '\n' || char_ > 127) { + return i; + } + } + + return text_len; +} + size_t IndexOfNewlineOrNonASCIIOrHashOrAtImpl(const uint8_t* HWY_RESTRICT start_ptr, size_t search_len) { ASSERT(search_len > 0); @@ -1294,6 +1338,7 @@ HWY_EXPORT(FirstNonAscii16Impl); HWY_EXPORT(FirstNonAscii8Impl); HWY_EXPORT(IndexOfAnyCharImpl); HWY_EXPORT(IndexOfCharImpl); +HWY_EXPORT(IndexOfInterestingCharacterInMultilineCommentImpl); HWY_EXPORT(IndexOfInterestingCharacterInStringLiteralImpl); HWY_EXPORT(IndexOfNeedsEscapeForJavaScriptStringImplBacktick); HWY_EXPORT(IndexOfNeedsEscapeForJavaScriptStringImplQuote); @@ -1375,6 +1420,11 @@ size_t highway_index_of_interesting_character_in_string_literal(const uint8_t* H return HWY_DYNAMIC_DISPATCH(IndexOfInterestingCharacterInStringLiteralImpl)(text, text_len, quote); } +size_t highway_index_of_interesting_character_in_multiline_comment(const uint8_t* HWY_RESTRICT text, size_t text_len) +{ + return HWY_DYNAMIC_DISPATCH(IndexOfInterestingCharacterInMultilineCommentImpl)(text, text_len); +} + size_t highway_index_of_newline_or_non_ascii(const uint8_t* HWY_RESTRICT haystack, size_t haystack_len) { return HWY_DYNAMIC_DISPATCH(IndexOfNewlineOrNonASCIIImpl)(haystack, haystack_len); diff --git a/test/bundler/bundler_comments.test.ts b/test/bundler/bundler_comments.test.ts index 540129c5ce9a..c05527343bf9 100644 --- a/test/bundler/bundler_comments.test.ts +++ b/test/bundler/bundler_comments.test.ts @@ -390,4 +390,43 @@ describe("multi-line comments", () => { expect(entry.originalLine).toBeGreaterThanOrEqual(0); }, }); + + // The lexer skips >=512-byte block comment bodies with SIMD; these verify + // large comments end-to-end (legal comment preservation, ASI, output code). + itBundled("large legal comment is preserved and does not corrupt the code after it", { + files: { + "/entry.js": + "/*!\r\n" + + " * Legal header line with some padding text to make the comment large enough.\r\n".repeat(20) + + " * Licensed under the ünïcödé license 🦊\r\n" + + " */\r\n" + + 'console.log("hello");', + }, + run: { + stdout: "hello", + }, + onAfterBundle(api) { + const output = api.readFile("/out.js"); + expect(output).toContain("Legal header line with some padding text"); + expect(output).toContain("Licensed under the ünïcödé license 🦊"); + }, + }); + + itBundled("newline inside a large block comment triggers ASI", { + files: { + "/entry.js": `function f() { return /*${Buffer.alloc(600, "x").toString()}\n${Buffer.alloc(600, "y").toString()}*/ "value" }\nconsole.log(String(f()));`, + }, + run: { + stdout: "undefined", + }, + }); + + itBundled("no ASI when a large block comment contains no newline", { + files: { + "/entry.js": `function f() { return /*${Buffer.alloc(1200, "x").toString()}*/ "value" }\nconsole.log(String(f()));`, + }, + run: { + stdout: "value", + }, + }); }); diff --git a/test/bundler/transpiler/transpiler.test.js b/test/bundler/transpiler/transpiler.test.js index ca9fd2a5d6bf..12517e5bfb18 100644 --- a/test/bundler/transpiler/transpiler.test.js +++ b/test/bundler/transpiler/transpiler.test.js @@ -4655,3 +4655,90 @@ describe("parse error flood", () => { expect(exitCode).toBe(0); }, 90_000); }); + +describe("multi-line comment scanning", () => { + // The lexer's block-comment scanner switches to a SIMD skip + // (bun_highway::index_of_interesting_character_in_multiline_comment) when at + // least 512 bytes of input remain, so cover sizes on both sides of that + // threshold and around vector-width boundaries, plus every byte class the + // skip has to stop at ('*', '\r', '\n', non-ASCII). + const transpiler = new Bun.Transpiler({ loader: "js" }); + const xPad = Buffer.alloc(8193, "x").toString(); + const pad600 = xPad.slice(0, 600); + + const expectParseError = (code, message) => { + try { + transpiler.transformSync(code); + } catch (er) { + let err = er; + if (er instanceof AggregateError) { + err = er.errors[0]; + } + expect(err.message).toBe(message); + return; + } + throw new Error("Expected parse error for code\n\t" + code); + }; + + it("strips block comments of every size around the SIMD threshold", () => { + const sizes = []; + for (let size = 480; size <= 576; size++) sizes.push(size); + sizes.push(1000, 4095, 4096, 4097, 8193); + for (const size of sizes) { + const out = transpiler.transformSync(`/*${xPad.slice(0, size)}*/ pass();`); + expect({ size, out }).toEqual({ size, out: "pass();\n" }); + } + }); + + it("handles a lone '*' at every offset inside a large comment", () => { + const aPad = Buffer.alloc(80, "a").toString(); + const bPad = Buffer.alloc(700, "b").toString(); + for (let offset = 0; offset < 80; offset++) { + const body = aPad.slice(0, offset) + "*" + bPad.slice(0, 700 - offset); + const out = transpiler.transformSync(`/*${body}*/ pass();`); + expect({ offset, out }).toEqual({ offset, out: "pass();\n" }); + } + }); + + it("handles comments made entirely of '*'", () => { + const stars = Buffer.alloc(600, "*").toString(); + expect(transpiler.transformSync(`/*${stars}*/ pass();`)).toBe("pass();\n"); + expect(transpiler.transformSync(`/*${stars}/ pass();`)).toBe("pass();\n"); + }); + + it("treats newlines inside a large block comment as line terminators (ASI)", () => { + for (const newline of ["\n", "\r", "\r\n", "\u2028", "\u2029"]) { + const out = transpiler.transformSync(`function f() { return /*${pad600}${newline}${pad600}*/ 1 }`); + expect({ newline, out }).toEqual({ newline, out: "function f() {\n return;\n}\n" }); + } + // control: no newline anywhere inside the comment, so no ASI + expect(transpiler.transformSync(`function f() { return /*${pad600}${pad600}*/ 1 }`)).toBe( + "function f() {\n return 1;\n}\n", + ); + }); + + it("scans large comments containing non-ASCII text", () => { + expect(transpiler.transformSync(`/*${"é".repeat(400)}*/ pass();`)).toBe("pass();\n"); + expect(transpiler.transformSync(`/*${"🦊".repeat(200)}*/ pass();`)).toBe("pass();\n"); + expect(transpiler.transformSync(`/* ${pad600} 日本語のコメント ${pad600} */ pass();`)).toBe("pass();\n"); + }); + + it("does not corrupt code around a large comment", () => { + const dashes = Buffer.alloc(700, "-").toString(); + const out = transpiler.transformSync(`const a = "before";/*${dashes}*/const b = "after"; console.log(a, b);`); + expect(out).toBe('const a = "before";\nconst b = "after";\nconsole.log(a, b);\n'); + }); + + it("handles a large comment that ends exactly at EOF", () => { + expect(transpiler.transformSync(`pass(); /*${pad600}*/`)).toBe("pass();\n"); + expect(transpiler.transformSync(`pass(); /*${pad600}**/`)).toBe("pass();\n"); + }); + + it("reports unterminated large block comments", () => { + const message = 'Expected "*/" to terminate multi-line comment'; + expectParseError(`/*${pad600}`, message); + expectParseError(`/*${pad600}*`, message); + expectParseError(`/*${Buffer.alloc(600, "*").toString()}`, message); + expectParseError(`/*${pad600}🦊`, message); + }); +}); diff --git a/test/js/bun/transpiler/transpiler-truncated-utf8-fixture.ts b/test/js/bun/transpiler/transpiler-truncated-utf8-fixture.ts index c6d53281c71d..61463fc800e3 100644 --- a/test/js/bun/transpiler/transpiler-truncated-utf8-fixture.ts +++ b/test/js/bun/transpiler/transpiler-truncated-utf8-fixture.ts @@ -55,6 +55,13 @@ const cases: Array<[string, number[]]> = [ ["4-byte lead + 1 continuation", [0x31, 0x40, 0xf0, 0x90]], ["4-byte lead + 2 continuations", [0x31, 0x40, 0xf0, 0x90, 0x80]], ["sourceMappingURL pragma + 4-byte lead", [...Buffer.from("//# sourceMappingURL=a"), 0xf0]], + // Block comments with >=512 bytes remaining go through the SIMD skip in the + // lexer (bun_highway::index_of_interesting_character_in_multiline_comment); + // ending the input exactly at the guard page catches any read past the end. + ["block comment terminated at buffer end", [...Buffer.from("x=1/*"), ...Buffer.alloc(700, 0x63), ...Buffer.from("*/")]], + ["unterminated block comment at buffer end", [...Buffer.from("x=1/*"), ...Buffer.alloc(700, 0x63)]], + ["unterminated block comment + 4-byte lead", [...Buffer.from("x=1/*"), ...Buffer.alloc(700, 0x63), 0xf0]], + ["unterminated block comment + '*'", [...Buffer.from("x=1/*"), ...Buffer.alloc(700, 0x63), 0x2a]], ]; for (const [name, bytes] of cases) { diff --git a/test/js/bun/transpiler/transpiler-truncated-utf8.test.ts b/test/js/bun/transpiler/transpiler-truncated-utf8.test.ts index 87f3e5a404c5..95a95980216a 100644 --- a/test/js/bun/transpiler/transpiler-truncated-utf8.test.ts +++ b/test/js/bun/transpiler/transpiler-truncated-utf8.test.ts @@ -32,6 +32,10 @@ describe.skipIf(!(isLinux || isMacOS))("Bun.Transpiler.transformSync with trunca expect.stringContaining("ok: 4-byte lead + 1 continuation"), expect.stringContaining("ok: 4-byte lead + 2 continuations"), expect.stringContaining("ok: sourceMappingURL pragma + 4-byte lead"), + expect.stringContaining("ok: block comment terminated at buffer end"), + expect.stringContaining("ok: unterminated block comment at buffer end"), + expect.stringContaining("ok: unterminated block comment + 4-byte lead"), + expect.stringContaining("ok: unterminated block comment + '*'"), "DONE", ], stderr: "",