From 6f088e2320d5b767848bff2b19c9987c07cf16e5 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 01:20:34 -0700 Subject: [PATCH 01/14] yaml: clear remaining 3 non-cyclic test.todo entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (1) [206] BOM strip in l-document-prefix: new `Encoding::bom_len()` returns 3 for UTF-8 EF BB BF, 1 for UTF-16 FEFF, 0 for Latin-1. `Parser::init` starts pos and line_start_pos past the BOM. (2) `&outer\n&inner : x` (e-node key, valid [200]/[193] split): clear `has_mapping_anchor` alongside `has_anchor` after implicit_key_anchors consumed both. `set_anchor` now errors instead of silently overwriting on a third anchor. (3) `{a: &x\n b: c}` (multiline-property cmi gap): the Scalar arm returns the bare scalar in `FlowIn && !flow_pair_allowed` regardless of the cmi indent comparison; the leftover `:` reaches the [140] check. 7 → 4 todos; the remaining 4 are cyclic anchors (deferred). --- src/parsers/yaml.rs | 62 +++++++++++++++++++++++++++++------ test/js/bun/yaml/yaml.test.ts | 25 +++++++------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 5851f585546..fd576ef773b 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -463,6 +463,9 @@ pub trait Encoding: Copy + 'static { /// buffer (see `EncLit` doc for the const-generics rationale). fn literal(s: &'static [u8]) -> EncLit; + /// Number of leading units to skip if `input` starts with [3] c-byte-order-mark. + fn bom_len(input: &[Self::Unit]) -> usize; + /// Reinterpret a `&[Unit]` slice as `&[u8]` for `StringHashMap` keying /// (`anchors` / `tag_handles`). Zig's `bun.StringHashMap` is keyed by /// `[]const u8`; calls like `tag_handles.put(handle.slice(self.input), {})` @@ -523,6 +526,10 @@ impl Encoding for Latin1 { // Only reachable from `EncodingKind::Utf16`-gated arms. unreachable!("unit_from_u16 on Latin1") } + #[inline] + fn bom_len(_input: &[u8]) -> usize { + 0 + } } impl Encoding for Utf8 { @@ -551,6 +558,14 @@ impl Encoding for Utf8 { // Only reachable from `EncodingKind::Utf16`-gated arms. unreachable!("unit_from_u16 on Utf8") } + #[inline] + fn bom_len(input: &[u8]) -> usize { + if input.len() >= 3 && input[0] == 0xEF && input[1] == 0xBB && input[2] == 0xBF { + 3 + } else { + 0 + } + } } impl Encoding for Utf16 { @@ -589,6 +604,14 @@ impl Encoding for Utf16 { u } #[inline] + fn bom_len(input: &[u16]) -> usize { + if input.first() == Some(&0xFEFF) { + 1 + } else { + 0 + } + } + #[inline] fn as_u16_slice(s: &[u16]) -> &[u16] { s } @@ -2359,11 +2382,13 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { pub const MAX_ALIAS_EXPANSION: usize = 16 * 1024 * 1024; pub fn init(bump: &'i bun_alloc::Arena, input: &'i [Enc::Unit]) -> Self { + // [206] l-document-prefix ::= c-byte-order-mark? l-comment* + let start = Pos::from(Enc::bom_len(input)); Self { input, bump, - pos: Pos::from(0), - line_start_pos: Pos::from(0), + pos: start, + line_start_pos: start, line_indent: Indent::NONE, tab_after_indent: false, line: Line::from(1), @@ -3467,7 +3492,9 @@ impl NodeProperties { pub fn set_anchor(&mut self, anchor_token: Token) -> Result<(), ParseError> { if let Some(previous_anchor) = &self.has_anchor { - if previous_anchor.line == anchor_token.line { + if previous_anchor.line == anchor_token.line + || self.has_mapping_anchor.is_some() + { return Err(ParseError::MultipleAnchors); } self.has_mapping_anchor = Some(previous_anchor.clone()); @@ -4234,14 +4261,13 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { self.anchors .put(Enc::key_bytes(mapping_anchor.slice(self.input)), mapping)?; } - // Clear has_anchor so the post-loop fallback doesn't - // re-register the inner anchor on the mapping. The - // remaining fields reach the post-loop guards: this - // matches main's behavior (rejects `!!a\n!!b\n: x` and - // `&a\n&b\n&c : x`), but also over-rejects the valid - // `&outer\n&inner : x` — pre-existing; the Scalar arm - // avoids it via `return Ok`. + // Anchors are fully consumed by implicit_key_anchors; + // clear both so the post-loop fallback doesn't re-register + // (or over-reject `&outer\n&inner : x` via the + // has_mapping_anchor guard). Tag fields stay so the + // has_mapping_tag guard still catches `!!a\n!!b\n: x`. node_props.has_anchor = None; + node_props.has_mapping_anchor = None; break 'node mapping; } @@ -4325,6 +4351,22 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { } } + // [147] flow-map value is ns-flow-node, not a pair. + // Return the bare scalar; the leftover `:` reaches the + // caller's [140] check. The cmi==scalar_indent path + // above already handles the same-line case; this + // covers the multiline-property case where they + // diverge. + if self.context.get() == Context::FlowIn + && !opts.flow_pair_allowed + { + break 'node scalar.data.to_expr( + scalar_start, + self.input, + self.bump, + ); + } + let implicit_key = scalar.data.to_expr(scalar_start, self.input, self.bump); let implicit_key_anchors = node_props.implicit_key_anchors(scalar_line)?; diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 7769f01a86e..16c27487825 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1214,7 +1214,7 @@ folded: > expect(() => YAML.parse("!!str\n!!map\n: x\n")).toThrow("Multiple tags"); }); - test.todo("two anchors before e-node `:` (outer=mapping, inner=key) — pre-existing over-reject", () => { + test("two anchors before e-node `:` (outer=mapping, inner=key)", () => { // Valid per [200]/[193]: outer anchors the collection, inner the // e-node key. The Scalar-key analogue (`&outer\n&inner b: x`) is // accepted because that arm `return Ok(mapping)`, bypassing the @@ -1501,12 +1501,11 @@ folded: > expect(() => YAML.parse("{? a: b: c}\n")).toThrow("Unexpected token"); }); - test.todo("flow-map value with multiline c-ns-properties before nested pair", () => { - // The cmi mechanism for [147] uses the first token's indent; when an - // anchor/tag is on a different line than the scalar, indents diverge - // and the cmi check is bypassed. Pre-existing. - expect(() => YAML.parse("{a: &x\n b: c}\n")).toThrow(); - expect(() => YAML.parse("{a: !!str\n b: c}\n")).toThrow(); + test("flow-map value with multiline c-ns-properties before nested pair", () => { + // [147] flow-map value is ns-flow-node; the Scalar arm returns the + // bare scalar in FlowIn when !flow_pair_allowed, regardless of cmi. + expect(() => YAML.parse("{a: &x\n b: c}\n")).toThrow("Unexpected token"); + expect(() => YAML.parse("{a: !!str\n b: c}\n")).toThrow("Unexpected token"); }); test("flow-seq pair value on next line at column ≤ key indent", () => { @@ -1727,13 +1726,13 @@ folded: > expect(() => YAML.parse("a\n...\n... b\n")).toThrow("Unexpected token"); }); - test.todo("BOM-prefixed `---` recognized as doc marker", () => { - // [202] l-document-prefix admits c-byte-order-mark before - // c-directives-end. is_at_line_start() is false after the BOM - // (pos != line_start_pos). Pre-existing differently-wrong (was a - // spurious BOM-only first doc); proper fix is BOM strip in - // l-document-prefix, not special-casing here. + test("BOM-prefixed `---` recognized as doc marker", () => { + // [206] l-document-prefix ::= c-byte-order-mark? l-comment* expect(YAML.parse("\uFEFF---\na: 1\n")).toEqual({ a: 1 }); + expect(YAML.parse("\uFEFFa: 1\n")).toEqual({ a: 1 }); + expect(YAML.parse("\uFEFF")).toEqual(null); + // BOM mid-stream is not stripped (only [206] document-prefix). + expect(YAML.parse("a: \uFEFFx\n")).toEqual({ a: "\uFEFFx" }); }); }); From 8687f17079440374f0d2c6f8edd69e32602aeda9 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 08:22:42 +0000 Subject: [PATCH 02/14] [autofix.ci] apply automated fixes --- src/parsers/yaml.rs | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index fd576ef773b..3d4810c5014 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -605,11 +605,7 @@ impl Encoding for Utf16 { } #[inline] fn bom_len(input: &[u16]) -> usize { - if input.first() == Some(&0xFEFF) { - 1 - } else { - 0 - } + if input.first() == Some(&0xFEFF) { 1 } else { 0 } } #[inline] fn as_u16_slice(s: &[u16]) -> &[u16] { @@ -3492,9 +3488,7 @@ impl NodeProperties { pub fn set_anchor(&mut self, anchor_token: Token) -> Result<(), ParseError> { if let Some(previous_anchor) = &self.has_anchor { - if previous_anchor.line == anchor_token.line - || self.has_mapping_anchor.is_some() - { + if previous_anchor.line == anchor_token.line || self.has_mapping_anchor.is_some() { return Err(ParseError::MultipleAnchors); } self.has_mapping_anchor = Some(previous_anchor.clone()); @@ -4357,14 +4351,8 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { // above already handles the same-line case; this // covers the multiline-property case where they // diverge. - if self.context.get() == Context::FlowIn - && !opts.flow_pair_allowed - { - break 'node scalar.data.to_expr( - scalar_start, - self.input, - self.bump, - ); + if self.context.get() == Context::FlowIn && !opts.flow_pair_allowed { + break 'node scalar.data.to_expr(scalar_start, self.input, self.bump); } let implicit_key = scalar.data.to_expr(scalar_start, self.input, self.bump); From 544b779203d7197407a599a815e59a07cfe1edf0 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 10:52:03 -0700 Subject: [PATCH 03/14] =?UTF-8?q?yaml:=20=C2=A75.2=20byte-order-mark=20enc?= =?UTF-8?q?oding=20detection;=20fix=20`#`=20regression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML::parse now applies the spec §5.2 byte-pattern matrix on byte input (Buffer/Uint8Array): detects UTF-8/UTF-16LE/UTF-16BE (with or without BOM) and routes to the matching Parser; UTF-32 errors with a clear message. The transcoded UTF-16 buffer is arena-allocated so it shares the Expr tree's lifetime. Also: the BOM-aware `Parser::init` made the two `#` arms' `Pos::ZERO` start-of-input checks stale (`# comment\na: 1` errored). Both now use `is_at_line_start()` which is BOM-aware. --- src/parsers/yaml.rs | 95 ++++++++++++++++++++++++++++++----- test/js/bun/yaml/yaml.test.ts | 29 +++++++++++ 2 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 3d4810c5014..054b4726154 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -25,6 +25,32 @@ use bun_core::{self, StackCheck}; pub struct YAML; +/// Spec §5.2 byte-order-mark / null-byte encoding detection. +#[derive(Clone, Copy, PartialEq, Eq)] +enum DetectedEncoding { + Utf8, + Utf16Le, + Utf16Be, + Utf32Le, + Utf32Be, +} + +fn detect_encoding(bytes: &[u8]) -> DetectedEncoding { + use DetectedEncoding::*; + let b = |i: usize| bytes.get(i).copied().unwrap_or(0xFF); + match (b(0), b(1), b(2), b(3)) { + (0x00, 0x00, 0xFE, 0xFF) => Utf32Be, + (0xFF, 0xFE, 0x00, 0x00) => Utf32Le, + (0x00, 0x00, 0x00, _) => Utf32Be, + (_, 0x00, 0x00, 0x00) => Utf32Le, + (0xFE, 0xFF, ..) => Utf16Be, + (0xFF, 0xFE, ..) => Utf16Le, + (0x00, _, ..) => Utf16Be, + (_, 0x00, ..) => Utf16Le, + _ => Utf8, + } +} + impl YAML { pub fn parse( source: &bun_ast::Source, @@ -34,12 +60,54 @@ impl YAML { // Zig: `bun.analytics.Features.yaml_parse += 1;` bun_core::analytics::Features::yaml_parse_inc(); - let mut parser: Parser = Parser::init(bump, source.contents()); + let bytes = source.contents(); + match detect_encoding(bytes) { + DetectedEncoding::Utf8 => Self::parse_with::(source, bytes, log, bump), + enc @ (DetectedEncoding::Utf16Le | DetectedEncoding::Utf16Be) => { + let be = matches!(enc, DetectedEncoding::Utf16Be); + if bytes.len() % 2 != 0 { + log.add_error( + Some(source), + bun_ast::Loc::EMPTY, + b"UTF-16 input has odd byte length", + ); + return Err(YamlParseError::SyntaxError); + } + // The Expr tree may hold slices into the input; allocate the + // transcoded buffer in the same arena so it shares its lifetime. + let units: &mut [u16] = bump.alloc_slice_fill_default(bytes.len() / 2); + for (i, c) in bytes.chunks_exact(2).enumerate() { + units[i] = if be { + u16::from_be_bytes([c[0], c[1]]) + } else { + u16::from_le_bytes([c[0], c[1]]) + }; + } + Self::parse_with::(source, units, log, bump) + } + DetectedEncoding::Utf32Le | DetectedEncoding::Utf32Be => { + log.add_error( + Some(source), + bun_ast::Loc::EMPTY, + b"UTF-32 input is not supported", + ); + Err(YamlParseError::SyntaxError) + } + } + } + + fn parse_with( + source: &bun_ast::Source, + input: &[Enc::Unit], + log: &mut bun_ast::Log, + bump: &bun_alloc::Arena, + ) -> Result { + let mut parser: Parser = Parser::init(bump, input); let stream = match parser.parse() { Ok(s) => s, Err(e) => { - let err = ParseResult::::fail(e, &parser); + let err = ParseResult::::fail(e, &parser); if let ParseResult::Err(err) = err { err.add_to_log(source, log)?; } @@ -4605,9 +4673,11 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { } 0x23 /* '#' */ => { - let prev = parser!().input[parser!().pos.sub(1).cast()]; - if parser!().pos == Pos::ZERO - || matches!(Enc::wide(prev), 0x20 | 0x09 | 0x0D | 0x0A) + if parser!().is_at_line_start() + || matches!( + Enc::wide(parser!().input[parser!().pos.sub(1).cast()]), + 0x20 | 0x09 | 0x0D | 0x0A + ) { return Ok(ctx.done()); } @@ -5887,13 +5957,14 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { } 0x23 /* '#' */ => { let start = self.pos; - let prev = if start == Pos::ZERO { 0 } else { Enc::wide(self.input[start.cast() - 1]) }; - match prev { - 0 | 0x20 | 0x09 | 0x0A | 0x0D => {} - _ => { - // TODO: prove this is unreachable - return Err(ParseError::UnexpectedCharacter); - } + if !self.is_at_line_start() + && !matches!( + Enc::wide(self.input[start.cast() - 1]), + 0x20 | 0x09 | 0x0A | 0x0D + ) + { + // TODO: prove this is unreachable + return Err(ParseError::UnexpectedCharacter); } self.inc(1); while !self.is_b_char_or_eof() { diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 16c27487825..988cb1a3c97 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1730,10 +1730,39 @@ folded: > // [206] l-document-prefix ::= c-byte-order-mark? l-comment* expect(YAML.parse("\uFEFF---\na: 1\n")).toEqual({ a: 1 }); expect(YAML.parse("\uFEFFa: 1\n")).toEqual({ a: 1 }); + expect(YAML.parse("\uFEFF# comment\na: 1\n")).toEqual({ a: 1 }); expect(YAML.parse("\uFEFF")).toEqual(null); // BOM mid-stream is not stripped (only [206] document-prefix). expect(YAML.parse("a: \uFEFFx\n")).toEqual({ a: "\uFEFFx" }); }); + + test("\u00A75.2 byte-order-mark encoding detection on byte input", () => { + const u16be = (s: string) => { + const b = Buffer.from(s, "utf16le"); + for (let i = 0; i < b.length; i += 2) [b[i], b[i + 1]] = [b[i + 1], b[i]]; + return b; + }; + // UTF-8 (with and without BOM) + expect(YAML.parse(Buffer.from("a: 1\n"))).toEqual({ a: 1 }); + expect(YAML.parse(Buffer.from([0xef, 0xbb, 0xbf, ...Buffer.from("a: 1\n")]))).toEqual({ a: 1 }); + // UTF-16LE / UTF-16BE (with BOM) + expect(YAML.parse(Buffer.from("\uFEFFa: 1\n", "utf16le"))).toEqual({ a: 1 }); + expect(YAML.parse(u16be("\uFEFFa: 1\n"))).toEqual({ a: 1 }); + // UTF-16LE / UTF-16BE (no BOM \u2014 detected via null-byte pattern) + expect(YAML.parse(Buffer.from("a: 1\n", "utf16le"))).toEqual({ a: 1 }); + expect(YAML.parse(u16be("a: 1\n"))).toEqual({ a: 1 }); + // UTF-32 \u2014 unsupported + expect(() => YAML.parse(Buffer.from([0, 0, 0xfe, 0xff, 0, 0, 0, 0x61]))).toThrow( + "UTF-32 input is not supported", + ); + expect(() => YAML.parse(Buffer.from([0xff, 0xfe, 0, 0, 0x61, 0, 0, 0]))).toThrow( + "UTF-32 input is not supported", + ); + // Odd byte count for UTF-16 + expect(() => YAML.parse(Buffer.from([0xff, 0xfe, 0x61]))).toThrow("UTF-16 input has odd byte length"); + // UTF-16 with non-ASCII content + expect(YAML.parse(Buffer.from("\uFEFFmsg: h\u00E9llo\n", "utf16le"))).toEqual({ msg: "h\u00E9llo" }); + }); }); describe("flow comma/separator placement", () => { From 6187d11cecfebed628122668511a638748ae6060 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 11:00:34 -0700 Subject: [PATCH 04/14] =?UTF-8?q?yaml:=20review=20fixes=20=E2=80=94=20mirr?= =?UTF-8?q?or=20[147]=20FlowIn=20early-return=20to=20Alias/SeqStart/MapSta?= =?UTF-8?q?rt;=20tests=20+=20tag-analogue=20todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cmi-gap fix in the Scalar arm (`FlowIn && !flow_pair_allowed → return bare`) is now mirrored to the Alias/SequenceStart/MappingStart arms so `{a: &x\n [b]: c}` etc. error instead of forming nested mappings. Adds test for the 3rd-anchor overflow guard, and a test.todo for the tag analogue (set_tag lacks the same overflow guard; has_mapping_tag isn't cleared in MappingValue arm). --- src/parsers/yaml.rs | 15 +++++++++++++++ test/js/bun/yaml/yaml.test.ts | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 054b4726154..4961d97c371 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -4066,6 +4066,11 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { } } + // [147] flow-map value is ns-flow-node, not a pair. + if self.context.get() == Context::FlowIn && !opts.flow_pair_allowed { + return Ok(copy); + } + let map = self.parse_block_mapping( copy, alias_start, @@ -4117,6 +4122,11 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { } } + // [147] flow-map value is ns-flow-node, not a pair. + if self.context.get() == Context::FlowIn && !opts.flow_pair_allowed { + break 'node seq; + } + let implicit_key_anchors = node_props.implicit_key_anchors(sequence_line)?; @@ -4204,6 +4214,11 @@ impl<'i, Enc: Encoding> Parser<'i, Enc> { } } + // [147] flow-map value is ns-flow-node, not a pair. + if self.context.get() == Context::FlowIn && !opts.flow_pair_allowed { + break 'node map; + } + let implicit_key_anchors = node_props.implicit_key_anchors(mapping_line)?; if let Some(key_anchor) = implicit_key_anchors.key_anchor { diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 988cb1a3c97..495dd55976d 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1357,6 +1357,18 @@ folded: > expect(() => YAML.parse("a: &x\n &y\n c: 1\n")).toThrow("Multiple anchors"); // Both props more-indented; inner same-line as key — valid. expect(YAML.parse("a:\n &x\n &y c: 1\n")).toEqual({ a: { c: 1 } }); + // 3rd anchor on a 3rd line — set_anchor's overflow guard catches it. + expect(() => YAML.parse("&a\n&b\n&c d: 1\n")).toThrow("Multiple anchors"); + }); + + test.todo("two tags before e-node `:` — [200]/[193] line split (tag analogue)", () => { + // The MappingValue arm clears has_anchor/has_mapping_anchor but not + // has_mapping_tag, so the post-loop guard over-rejects the valid + // 2-tag case. set_tag also lacks the has_mapping_tag.is_some() + // overflow guard that set_anchor has, so a 3rd tag on a 3rd line + // silently overwrites instead of erroring. + expect(YAML.parse("!!map\n!!str : x\n")).toEqual({ "": "x" }); + expect(() => YAML.parse("!!a\n!!b\n!!c d: 1\n")).toThrow("Multiple tags"); }); }); @@ -1513,6 +1525,10 @@ folded: > // before the value at any indentation is valid. expect(YAML.parse('["a":\nb]\n')).toEqual([{ a: "b" }]); expect(YAML.parse("[a: \nb]\n")).toEqual([{ a: "b" }]); + // Mirrored to Alias/SequenceStart/MappingStart arms. + expect(() => YAML.parse("{a: &x\n [b]: c}\n")).toThrow("Unexpected token"); + expect(() => YAML.parse("{a: &x\n {b}: c}\n")).toThrow("Unexpected token"); + expect(() => YAML.parse("{x: &y 1, a: &z\n *y : c}\n")).toThrow("Unexpected token"); }); test("multiline JSON-style key check ordering", () => { From 68858528403e270f52d148629e429161446b90b8 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 11:25:51 -0700 Subject: [PATCH 05/14] yaml: validate Core schema number regex before parse_double MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The number-scanning loop accepted `+`/`-`/`e`/`.` at any position, then `wtf::parse_double` prefix-parsed — so `1+1`, `++1`, `1e`, `1e1.5` all resolved as numbers (1, 1, 1, 10). Now `is_core_schema_number` validates the consumed slice against the [10.2.1.4] regex before calling parse_double; non-matching scalars stay as strings. Also fixes pre-existing `-.5`/`+.5` → string (the loop's `decimal` pre-check + no-inc-for-sign meant the `.` was seen as a second decimal). 41-case test matrix. --- src/parsers/yaml.rs | 69 +++++++++++++++++++++++++++++++++-- test/js/bun/yaml/yaml.test.ts | 31 ++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 4961d97c371..6f755c408e4 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -1563,7 +1563,12 @@ impl<'i, Enc: Encoding> ScalarResolverCtx<'i, Enc> { let mut minus = false; let mut hex = false; - if first_char != FirstChar::Negative && first_char != FirstChar::Positive { + // For Negative/Positive the sign was consumed by the caller; the first + // body char (digit or `.`) is at `pos`. For Other/Dot the caller left + // `pos` at the first body char too. Either way, advance past it so the + // loop starts at the second body char with the `decimal`/digit flags + // already reflecting the first. + if !matches!(first_char, FirstChar::Negative | FirstChar::Positive) || decimal { parser!().inc(1); } @@ -1717,15 +1722,23 @@ impl<'i, Enc: Encoding> ScalarResolverCtx<'i, Enc> { return Ok(()); } + let lexed = parser!().slice(start, end); let mut scalar: NodeScalar = 'scalar: { if x || o || hex { - let unsigned = match parse_unsigned_radix0::(parser!().slice(start, end)) { + let unsigned = match parse_unsigned_radix0::(lexed) { Ok(v) => v, Err(_) => return Ok(()), }; break 'scalar NodeScalar::Number(unsigned as f64); } - let float = match parse_double_generic::(parser!().slice(start, end)) { + // [10.2.1.4] Core schema float/int regex. The lexer loop above is + // permissive (accepts `+`/`-`/`e`/`.` at any position) and + // `wtf::parse_double` prefix-parses, so `1+1` would resolve as 1. + // Validate the consumed slice matches the schema before parsing. + if !is_core_schema_number::(lexed, first_char) { + return Ok(()); + } + let float = match parse_double_generic::(lexed) { Ok(v) => v, Err(_) => return Ok(()), }; @@ -1751,6 +1764,56 @@ impl<'i, Enc: Encoding> ScalarResolverCtx<'i, Enc> { } } +/// [10.2.1.4] Core schema int/float pattern. The slice may already have had a +/// leading `.` or `+`/`-` consumed by the caller before `start` was captured; +/// `first_char` carries that. +/// `[-+]? ( \. [0-9]+ | [0-9]+ ( \. [0-9]* )? ) ( [eE] [-+]? [0-9]+ )?` +fn is_core_schema_number(s: &[Enc::Unit], first_char: FirstChar) -> bool { + let mut i = 0usize; + let len = s.len(); + let at = |j: usize| Enc::wide(s[j]); + let is_digit = |c: u32| (0x30..=0x39).contains(&c); + + // Mantissa: \. [0-9]+ | [0-9]+ ( \. [0-9]* )? + let saw_leading_dot = first_char == FirstChar::Dot + || (i < len && at(i) == 0x2E && { i += 1; true }); + if saw_leading_dot { + if i >= len || !is_digit(at(i)) { + return false; + } + while i < len && is_digit(at(i)) { + i += 1; + } + } else { + if i >= len || !is_digit(at(i)) { + return false; + } + while i < len && is_digit(at(i)) { + i += 1; + } + if i < len && at(i) == 0x2E { + i += 1; + while i < len && is_digit(at(i)) { + i += 1; + } + } + } + // Optional exponent: [eE] [-+]? [0-9]+ + if i < len && matches!(at(i), 0x65 | 0x45) { + i += 1; + if i < len && matches!(at(i), 0x2B | 0x2D) { + i += 1; + } + if i >= len || !is_digit(at(i)) { + return false; + } + while i < len && is_digit(at(i)) { + i += 1; + } + } + i == len +} + /// Port of `bun.jsc.wtf.parseDouble(slice)` over an encoding-generic slice. /// `bun_core::wtf::parse_double` takes `&[u8]`; for `Utf8`/`Latin1` we narrow /// via `Enc::key_bytes` (identity). For `Utf16` the lexer guarantees the diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 495dd55976d..887fec153ff 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1781,6 +1781,37 @@ folded: > }); }); + describe("[10.2.1.4] Core schema number resolution", () => { + // The lexer loop accepted `+`/`-`/`e`/`.` at any position and + // wtf::parse_double prefix-parses, so `1+1` resolved as 1. Validation + // now requires the consumed slice to match the Core schema regex. + test.each([ + ["1+1", "1+1"], ["1-1", "1-1"], ["++1", "++1"], ["--1", "--1"], ["+-1", "+-1"], + ["1e", "1e"], ["1e+", "1e+"], ["1e-", "1e-"], ["1E", "1E"], + ["1e1.5", "1e1.5"], ["1e.5", "1e.5"], + ["1.2.3", "1.2.3"], [".", "."], [".e5", ".e5"], ["-.e5", "-.e5"], ["e5", "e5"], + ] as const)("%s resolves as string", (input, expected) => { + expect(Bun.YAML.parse(input)).toBe(expected); + }); + test.each([ + ["1", 1], ["-1", -1], ["+1", 1], ["0", 0], ["-0", -0], + ["1.5", 1.5], [".5", 0.5], ["-.5", -0.5], ["+.5", 0.5], ["1.", 1], + ["1e5", 1e5], ["1e+5", 1e5], ["1e-5", 1e-5], ["1.5e10", 1.5e10], + [".5e2", 50], ["-.5e2", -50], + ["0x1f", 31], ["0o17", 15], ["-0x1f", -31], + ] as const)("%s resolves as number %p", (input, expected) => { + expect(Bun.YAML.parse(input)).toBe(expected); + }); + test.each([".inf", "-.inf", "+.inf", ".Inf", ".INF"])("%s resolves as Infinity", input => { + expect(Math.abs(Bun.YAML.parse(input))).toBe(Infinity); + }); + test(".nan resolves as NaN", () => { + expect(Bun.YAML.parse(".nan")).toBeNaN(); + expect(Bun.YAML.parse(".NaN")).toBeNaN(); + expect(Bun.YAML.parse(".NAN")).toBeNaN(); + }); + }); + describe("flow comma/separator placement", () => { test("JSON-adjacent does not apply in flow-map value position", () => { // [147] flow-map value is ns-flow-node, not ns-flow-pair; [140] From 86ec4aad07e664c828fc31bdef788add952e0562 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 18:27:55 +0000 Subject: [PATCH 06/14] [autofix.ci] apply automated fixes --- src/parsers/yaml.rs | 5 +++- test/js/bun/yaml/yaml.test.ts | 44 ++++++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 6f755c408e4..b71f63d786b 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -1776,7 +1776,10 @@ fn is_core_schema_number(s: &[Enc::Unit], first_char: FirstChar) // Mantissa: \. [0-9]+ | [0-9]+ ( \. [0-9]* )? let saw_leading_dot = first_char == FirstChar::Dot - || (i < len && at(i) == 0x2E && { i += 1; true }); + || (i < len && at(i) == 0x2E && { + i += 1; + true + }); if saw_leading_dot { if i >= len || !is_digit(at(i)) { return false; diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 887fec153ff..15f524388d9 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1786,19 +1786,45 @@ folded: > // wtf::parse_double prefix-parses, so `1+1` resolved as 1. Validation // now requires the consumed slice to match the Core schema regex. test.each([ - ["1+1", "1+1"], ["1-1", "1-1"], ["++1", "++1"], ["--1", "--1"], ["+-1", "+-1"], - ["1e", "1e"], ["1e+", "1e+"], ["1e-", "1e-"], ["1E", "1E"], - ["1e1.5", "1e1.5"], ["1e.5", "1e.5"], - ["1.2.3", "1.2.3"], [".", "."], [".e5", ".e5"], ["-.e5", "-.e5"], ["e5", "e5"], + ["1+1", "1+1"], + ["1-1", "1-1"], + ["++1", "++1"], + ["--1", "--1"], + ["+-1", "+-1"], + ["1e", "1e"], + ["1e+", "1e+"], + ["1e-", "1e-"], + ["1E", "1E"], + ["1e1.5", "1e1.5"], + ["1e.5", "1e.5"], + ["1.2.3", "1.2.3"], + [".", "."], + [".e5", ".e5"], + ["-.e5", "-.e5"], + ["e5", "e5"], ] as const)("%s resolves as string", (input, expected) => { expect(Bun.YAML.parse(input)).toBe(expected); }); test.each([ - ["1", 1], ["-1", -1], ["+1", 1], ["0", 0], ["-0", -0], - ["1.5", 1.5], [".5", 0.5], ["-.5", -0.5], ["+.5", 0.5], ["1.", 1], - ["1e5", 1e5], ["1e+5", 1e5], ["1e-5", 1e-5], ["1.5e10", 1.5e10], - [".5e2", 50], ["-.5e2", -50], - ["0x1f", 31], ["0o17", 15], ["-0x1f", -31], + ["1", 1], + ["-1", -1], + ["+1", 1], + ["0", 0], + ["-0", -0], + ["1.5", 1.5], + [".5", 0.5], + ["-.5", -0.5], + ["+.5", 0.5], + ["1.", 1], + ["1e5", 1e5], + ["1e+5", 1e5], + ["1e-5", 1e-5], + ["1.5e10", 1.5e10], + [".5e2", 50], + ["-.5e2", -50], + ["0x1f", 31], + ["0o17", 15], + ["-0x1f", -31], ] as const)("%s resolves as number %p", (input, expected) => { expect(Bun.YAML.parse(input)).toBe(expected); }); From e341f057c5bd30293cf3fea4a03b548280e11d72 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 11:32:45 -0700 Subject: [PATCH 07/14] yaml: add test.todo for 14 bug categories surfaced by multi-modal bughunt NUL truncation, C0/C1/DEL accepted, CRLF quoted-fold, verbatim/named tag resolution, tag-on-quoted coercion, !!int/!!float validation, surrogate- pair escapes, s-separate after tag, directive validation, 1024-char implicit-key limit, error position discarded, merge-key order, alias with prior-line property, block-scalar header whitespace. Each todo asserts the spec-correct result with a [N] citation. --- test/js/bun/yaml/yaml.test.ts | 95 +++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 15f524388d9..c9aa7773368 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1838,6 +1838,101 @@ folded: > }); }); + // Bugs surfaced by the multi-modal bughunt (12 finder lenses × 3 rounds). + // Each todo asserts the spec-correct result. + describe("bughunt findings", () => { + test.todo("NUL byte (U+0000) is not c-printable — should error, not truncate", () => { + // [1] c-printable excludes NUL. Currently NUL is the EOF sentinel, so + // input is silently truncated. Data loss / security-adjacent. + expect(() => YAML.parse("a: 1\x00b: 2")).toThrow(); + expect(() => YAML.parse("key: foo\x00bar")).toThrow(); + }); + + test.todo("C0/C1/DEL control characters are not c-printable — should error", () => { + // [1] c-printable: x09, x0A, x0D, x20-x7E, x85, xA0-D7FF, E000-FFFD, + // 10000-10FFFF. Currently x01-x08/x0B/x0C/x0E-x1F/x7F/x80-x84/x86-x9F + // are accepted as scalar content. + expect(() => YAML.parse("a\x01b")).toThrow(); + expect(() => YAML.parse("a\x7Fb")).toThrow(); + expect(() => YAML.parse("a\x80b")).toThrow(); + }); + + test.todo("CRLF in quoted scalars folds as one line break (→ space)", () => { + // [73] b-l-folded: a single break folds to a space. Currently `\r\n` + // in quoted scalars produces `\n` instead. + expect(YAML.parse('"a\r\nb"')).toBe("a b"); + expect(YAML.parse("'a\r\nb'")).toBe("a b"); + }); + + test.todo("verbatim/named-handle tags resolve as Core-schema types", () => { + // [10.2] tag:yaml.org,2002:int via `!<...>` or `%TAG !y! ...` should + // resolve identically to `!!int`. Currently only the `!!` shorthand + // resolves. + expect(YAML.parse("! 42")).toBe(42); + expect(YAML.parse("%TAG !y! tag:yaml.org,2002:\n---\n!y!int 42")).toBe(42); + }); + + test.todo("explicit tag on quoted scalar coerces ([10.1.1.8] resolve via tag)", () => { + expect(YAML.parse("!!bool 'true'")).toBe(true); + expect(YAML.parse('!!int "42"')).toBe(42); + }); + + test.todo("`!!int`/`!!float` validate their content", () => { + // [10.2.1.2]/[10.2.1.4] — the tag's regex must match. + expect(() => YAML.parse("!!int 1.5")).toThrow(); + expect(() => YAML.parse("!!float 0x1f")).toThrow(); + }); + + test.todo("`\\uXXXX` surrogate pairs combine ([57] ns-esc-16-bit)", () => { + // js-yaml/eemeli combine surrogate halves to the supplementary code + // point. Currently rejected. + expect(YAML.parse('"\\uD834\\uDD1E"')).toBe("𝄞"); + }); + + test.todo("s-separate required after tag ([97] c-ns-tag-property)", () => { + // No whitespace between tag and content. + expect(() => YAML.parse("!b")).toThrow(); + expect(() => YAML.parse("!tag,x a")).toThrow(); + }); + + test.todo("`%YAML`/`%TAG` directive validation", () => { + // [86]/[88] require arguments; [87] requires major version 1; + // [89] forbids duplicate handle in same document. + expect(() => YAML.parse("%YAML\n---\nfoo")).toThrow(); + expect(() => YAML.parse("%YAML 2.0\n---\nfoo")).toThrow(); + expect(() => YAML.parse("%TAG !e! tag:a:\n%TAG !e! tag:b:\n---\nfoo")).toThrow(); + }); + + test.todo("§7.4.2 1024-char implicit-key limit enforced", () => { + const long = Buffer.alloc(1025, "a").toString(); + expect(() => YAML.parse(`${long}: v`)).toThrow(); + expect(() => YAML.parse(`[${long}: v]`)).toThrow(); + }); + + test.todo("error message includes line:col position", () => { + // ParseResultError carries Pos; the JS binding discards it. + try { YAML.parse("a: 1\nb:\n\tc: 2"); } catch (e: any) { + expect(e.message).toMatch(/line\s*\d|:\d+:\d+/); + } + }); + + test.todo("`<<:` merge preserves source property order", () => { + const r: any = YAML.parse("x: &x\n a: 1\n b: 2\n c: 3\ny:\n <<: *x"); + expect(Object.keys(r.y)).toEqual(["a", "b", "c"]); + }); + + test.todo("alias with property on prior line is rejected ([104] c-ns-alias-node)", () => { + // An alias node has no properties; a tag/anchor on a prior line + // belongs to a different node. + expect(() => YAML.parse("- &a 1\n- !!str\n *a")).toThrow(); + }); + + test.todo("block-scalar header rejects whitespace before chomp/indent indicator", () => { + // [162] c-b-block-header: no s-separate between `|`/`>` and indicators. + expect(() => YAML.parse("| 1\n text")).toThrow(); + }); + }); + describe("flow comma/separator placement", () => { test("JSON-adjacent does not apply in flow-map value position", () => { // [147] flow-map value is ns-flow-node, not ns-flow-pair; [140] From 755f18a99774d2e46fac6ce3ce465e1a1dc460f8 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 18:34:49 +0000 Subject: [PATCH 08/14] [autofix.ci] apply automated fixes --- test/js/bun/yaml/yaml.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index c9aa7773368..8dd3cbd7e62 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1911,7 +1911,9 @@ folded: > test.todo("error message includes line:col position", () => { // ParseResultError carries Pos; the JS binding discards it. - try { YAML.parse("a: 1\nb:\n\tc: 2"); } catch (e: any) { + try { + YAML.parse("a: 1\nb:\n\tc: 2"); + } catch (e: any) { expect(e.message).toMatch(/line\s*\d|:\d+:\d+/); } }); From 938fb32c072a49de851c6c5b08f49b4e1f988058 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 11:49:50 -0700 Subject: [PATCH 09/14] =?UTF-8?q?yaml:=20review=20nits=20=E2=80=94=20is=5F?= =?UTF-8?q?multiple=5Fof,=20Infinity=20sign=20asserts,=20UTF-16=20Loc=20TO?= =?UTF-8?q?DO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parsers/yaml.rs | 7 ++++++- test/js/bun/yaml/yaml.test.ts | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index b71f63d786b..8027c09774f 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -65,7 +65,7 @@ impl YAML { DetectedEncoding::Utf8 => Self::parse_with::(source, bytes, log, bump), enc @ (DetectedEncoding::Utf16Le | DetectedEncoding::Utf16Be) => { let be = matches!(enc, DetectedEncoding::Utf16Be); - if bytes.len() % 2 != 0 { + if !bytes.len().is_multiple_of(2) { log.add_error( Some(source), bun_ast::Loc::EMPTY, @@ -75,6 +75,11 @@ impl YAML { } // The Expr tree may hold slices into the input; allocate the // transcoded buffer in the same arena so it shares its lifetime. + // TODO: Parser tracks `pos` as a u16 index, but + // `Pos::loc()` is interpreted as a byte offset into + // `source.contents` — error positions for UTF-16 byte input are + // off. Fixing requires mapping u16 index → byte offset before + // constructing `Loc`. let units: &mut [u16] = bump.alloc_slice_fill_default(bytes.len() / 2); for (i, c) in bytes.chunks_exact(2).enumerate() { units[i] = if be { diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 8dd3cbd7e62..4a899d97fa0 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1828,8 +1828,11 @@ folded: > ] as const)("%s resolves as number %p", (input, expected) => { expect(Bun.YAML.parse(input)).toBe(expected); }); - test.each([".inf", "-.inf", "+.inf", ".Inf", ".INF"])("%s resolves as Infinity", input => { - expect(Math.abs(Bun.YAML.parse(input))).toBe(Infinity); + test.each([ + [".inf", Infinity], ["+.inf", Infinity], [".Inf", Infinity], [".INF", Infinity], + ["-.inf", -Infinity], ["-.Inf", -Infinity], ["-.INF", -Infinity], + ] as const)("%s resolves as %p", (input, expected) => { + expect(Bun.YAML.parse(input)).toBe(expected); }); test(".nan resolves as NaN", () => { expect(Bun.YAML.parse(".nan")).toBeNaN(); From fc12ed7acdd1b8b582e3a41f3ff0ebcf9595dd13 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 18:51:59 +0000 Subject: [PATCH 10/14] [autofix.ci] apply automated fixes --- test/js/bun/yaml/yaml.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 4a899d97fa0..f20ce65c131 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1829,8 +1829,13 @@ folded: > expect(Bun.YAML.parse(input)).toBe(expected); }); test.each([ - [".inf", Infinity], ["+.inf", Infinity], [".Inf", Infinity], [".INF", Infinity], - ["-.inf", -Infinity], ["-.Inf", -Infinity], ["-.INF", -Infinity], + [".inf", Infinity], + ["+.inf", Infinity], + [".Inf", Infinity], + [".INF", Infinity], + ["-.inf", -Infinity], + ["-.Inf", -Infinity], + ["-.INF", -Infinity], ] as const)("%s resolves as %p", (input, expected) => { expect(Bun.YAML.parse(input)).toBe(expected); }); From 89c9b9b78c13a2e4ec7c88f949a91dab8e40db42 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 11:57:58 -0700 Subject: [PATCH 11/14] =?UTF-8?q?yaml:=20clippy=20=E2=80=94=20drop=20redun?= =?UTF-8?q?dant=20`=5F,=20..`=20in=20detect=5Fencoding=20match?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parsers/yaml.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 8027c09774f..26786c17de9 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -43,10 +43,10 @@ fn detect_encoding(bytes: &[u8]) -> DetectedEncoding { (0xFF, 0xFE, 0x00, 0x00) => Utf32Le, (0x00, 0x00, 0x00, _) => Utf32Be, (_, 0x00, 0x00, 0x00) => Utf32Le, - (0xFE, 0xFF, ..) => Utf16Be, - (0xFF, 0xFE, ..) => Utf16Le, - (0x00, _, ..) => Utf16Be, - (_, 0x00, ..) => Utf16Le, + (0xFE, 0xFF, _, _) => Utf16Be, + (0xFF, 0xFE, _, _) => Utf16Le, + (0x00, ..) => Utf16Be, + (_, 0x00, _, _) => Utf16Le, _ => Utf8, } } From fcd70c042f6f6d0ce344d2dac55d2c93caa95619 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 12:28:51 -0700 Subject: [PATCH 12/14] =?UTF-8?q?yaml:=20review=20nits=20=E2=80=94=20split?= =?UTF-8?q?=20mirrored-arm=20asserts=20to=20own=20test;=20toThrow=20for=20?= =?UTF-8?q?line:col=20todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/js/bun/yaml/yaml.test.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index f20ce65c131..13ebdcd8d39 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1525,7 +1525,9 @@ folded: > // before the value at any indentation is valid. expect(YAML.parse('["a":\nb]\n')).toEqual([{ a: "b" }]); expect(YAML.parse("[a: \nb]\n")).toEqual([{ a: "b" }]); - // Mirrored to Alias/SequenceStart/MappingStart arms. + }); + + test("flow-map [147] value guard mirrored to Alias/SequenceStart/MappingStart arms", () => { expect(() => YAML.parse("{a: &x\n [b]: c}\n")).toThrow("Unexpected token"); expect(() => YAML.parse("{a: &x\n {b}: c}\n")).toThrow("Unexpected token"); expect(() => YAML.parse("{x: &y 1, a: &z\n *y : c}\n")).toThrow("Unexpected token"); @@ -1919,11 +1921,7 @@ folded: > test.todo("error message includes line:col position", () => { // ParseResultError carries Pos; the JS binding discards it. - try { - YAML.parse("a: 1\nb:\n\tc: 2"); - } catch (e: any) { - expect(e.message).toMatch(/line\s*\d|:\d+:\d+/); - } + expect(() => YAML.parse("a: 1\nb:\n\tc: 2")).toThrow(/line\s*\d|:\d+:\d+/); }); test.todo("`<<:` merge preserves source property order", () => { From 12c16e5fc090ecc55ad8b8813eb4778ff1bea2fd Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 12:44:57 -0700 Subject: [PATCH 13/14] =?UTF-8?q?yaml:=20revert=20=C2=A75.2=20encoding=20d?= =?UTF-8?q?etection=20=E2=80=94=20keep=20only=20UTF-8=20BOM=20strip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Parser` has never been exercised; introducing it (or erroring on detected UTF-16/32 byte input) trades one untested behavior for another. Restore main's `YAML::parse` exactly: `Parser` only. The `Encoding` trait/`bom_len()` infrastructure stays so a UTF-16 path can be added later behind a full test sweep, but no detection runs today. --- src/parsers/yaml.rs | 77 +---------------------------------- test/js/bun/yaml/yaml.test.ts | 25 +----------- 2 files changed, 3 insertions(+), 99 deletions(-) diff --git a/src/parsers/yaml.rs b/src/parsers/yaml.rs index 26786c17de9..1b7c197840f 100644 --- a/src/parsers/yaml.rs +++ b/src/parsers/yaml.rs @@ -25,32 +25,6 @@ use bun_core::{self, StackCheck}; pub struct YAML; -/// Spec §5.2 byte-order-mark / null-byte encoding detection. -#[derive(Clone, Copy, PartialEq, Eq)] -enum DetectedEncoding { - Utf8, - Utf16Le, - Utf16Be, - Utf32Le, - Utf32Be, -} - -fn detect_encoding(bytes: &[u8]) -> DetectedEncoding { - use DetectedEncoding::*; - let b = |i: usize| bytes.get(i).copied().unwrap_or(0xFF); - match (b(0), b(1), b(2), b(3)) { - (0x00, 0x00, 0xFE, 0xFF) => Utf32Be, - (0xFF, 0xFE, 0x00, 0x00) => Utf32Le, - (0x00, 0x00, 0x00, _) => Utf32Be, - (_, 0x00, 0x00, 0x00) => Utf32Le, - (0xFE, 0xFF, _, _) => Utf16Be, - (0xFF, 0xFE, _, _) => Utf16Le, - (0x00, ..) => Utf16Be, - (_, 0x00, _, _) => Utf16Le, - _ => Utf8, - } -} - impl YAML { pub fn parse( source: &bun_ast::Source, @@ -60,59 +34,12 @@ impl YAML { // Zig: `bun.analytics.Features.yaml_parse += 1;` bun_core::analytics::Features::yaml_parse_inc(); - let bytes = source.contents(); - match detect_encoding(bytes) { - DetectedEncoding::Utf8 => Self::parse_with::(source, bytes, log, bump), - enc @ (DetectedEncoding::Utf16Le | DetectedEncoding::Utf16Be) => { - let be = matches!(enc, DetectedEncoding::Utf16Be); - if !bytes.len().is_multiple_of(2) { - log.add_error( - Some(source), - bun_ast::Loc::EMPTY, - b"UTF-16 input has odd byte length", - ); - return Err(YamlParseError::SyntaxError); - } - // The Expr tree may hold slices into the input; allocate the - // transcoded buffer in the same arena so it shares its lifetime. - // TODO: Parser tracks `pos` as a u16 index, but - // `Pos::loc()` is interpreted as a byte offset into - // `source.contents` — error positions for UTF-16 byte input are - // off. Fixing requires mapping u16 index → byte offset before - // constructing `Loc`. - let units: &mut [u16] = bump.alloc_slice_fill_default(bytes.len() / 2); - for (i, c) in bytes.chunks_exact(2).enumerate() { - units[i] = if be { - u16::from_be_bytes([c[0], c[1]]) - } else { - u16::from_le_bytes([c[0], c[1]]) - }; - } - Self::parse_with::(source, units, log, bump) - } - DetectedEncoding::Utf32Le | DetectedEncoding::Utf32Be => { - log.add_error( - Some(source), - bun_ast::Loc::EMPTY, - b"UTF-32 input is not supported", - ); - Err(YamlParseError::SyntaxError) - } - } - } - - fn parse_with( - source: &bun_ast::Source, - input: &[Enc::Unit], - log: &mut bun_ast::Log, - bump: &bun_alloc::Arena, - ) -> Result { - let mut parser: Parser = Parser::init(bump, input); + let mut parser: Parser = Parser::init(bump, source.contents()); let stream = match parser.parse() { Ok(s) => s, Err(e) => { - let err = ParseResult::::fail(e, &parser); + let err = ParseResult::::fail(e, &parser); if let ParseResult::Err(err) = err { err.add_to_log(source, log)?; } diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 13ebdcd8d39..7ef7179890c 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1754,32 +1754,9 @@ folded: > expect(YAML.parse("a: \uFEFFx\n")).toEqual({ a: "\uFEFFx" }); }); - test("\u00A75.2 byte-order-mark encoding detection on byte input", () => { - const u16be = (s: string) => { - const b = Buffer.from(s, "utf16le"); - for (let i = 0; i < b.length; i += 2) [b[i], b[i + 1]] = [b[i + 1], b[i]]; - return b; - }; - // UTF-8 (with and without BOM) + test("UTF-8 BOM on byte input is stripped", () => { expect(YAML.parse(Buffer.from("a: 1\n"))).toEqual({ a: 1 }); expect(YAML.parse(Buffer.from([0xef, 0xbb, 0xbf, ...Buffer.from("a: 1\n")]))).toEqual({ a: 1 }); - // UTF-16LE / UTF-16BE (with BOM) - expect(YAML.parse(Buffer.from("\uFEFFa: 1\n", "utf16le"))).toEqual({ a: 1 }); - expect(YAML.parse(u16be("\uFEFFa: 1\n"))).toEqual({ a: 1 }); - // UTF-16LE / UTF-16BE (no BOM \u2014 detected via null-byte pattern) - expect(YAML.parse(Buffer.from("a: 1\n", "utf16le"))).toEqual({ a: 1 }); - expect(YAML.parse(u16be("a: 1\n"))).toEqual({ a: 1 }); - // UTF-32 \u2014 unsupported - expect(() => YAML.parse(Buffer.from([0, 0, 0xfe, 0xff, 0, 0, 0, 0x61]))).toThrow( - "UTF-32 input is not supported", - ); - expect(() => YAML.parse(Buffer.from([0xff, 0xfe, 0, 0, 0x61, 0, 0, 0]))).toThrow( - "UTF-32 input is not supported", - ); - // Odd byte count for UTF-16 - expect(() => YAML.parse(Buffer.from([0xff, 0xfe, 0x61]))).toThrow("UTF-16 input has odd byte length"); - // UTF-16 with non-ASCII content - expect(YAML.parse(Buffer.from("\uFEFFmsg: h\u00E9llo\n", "utf16le"))).toEqual({ msg: "h\u00E9llo" }); }); }); From efcfffe177c0304bb1e0e48761a7956296d6b725 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 May 2026 14:28:15 -0700 Subject: [PATCH 14/14] =?UTF-8?q?yaml:=20review=20nits=20=E2=80=94=20signe?= =?UTF-8?q?d-hex/octal=20as=20test.todo,=20YAML.parse=20import,=20stale=20?= =?UTF-8?q?comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `-0x1f`/`-0o17` (pre-existing on int path, not the float gate) → test.todo asserting the spec value, not codifying current -31 - `Bun.YAML.parse` → imported `YAML.parse` for consistency - Drop "Pre-existing on main" from un-todo'd test comment - Update string_is_number doc to reflect is_core_schema_number gate --- src/runtime/api/YAMLObject.rs | 18 ++++++++---------- test/js/bun/yaml/yaml.test.ts | 29 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/runtime/api/YAMLObject.rs b/src/runtime/api/YAMLObject.rs index 7b2b6422395..8cdf6649719 100644 --- a/src/runtime/api/YAMLObject.rs +++ b/src/runtime/api/YAMLObject.rs @@ -900,16 +900,14 @@ fn string_needs_quotes(str: &BunString) -> bool { /// Returns true when `str` would be parsed back as a number by `YAML.parse`. /// -/// This mirrors the rules in `src/interchange/yaml.zig`'s `tryResolveNumber`: -/// - Optional leading sign, optionally followed by `.inf`/`.Inf`/`.INF` for signed infinity. -/// - Otherwise a numeric mantissa: digits/`.`/`e`/`E`/hex letters, plus additional `+`/`-` -/// (the parser accepts any number of `+` after the leading sign as long as no `x` was -/// seen, and at most one additional `-`). -/// - `0x` / `0X` → hex digits; `0o` / `0O` → octal digits. -/// - Additionally, `wtf.parseDouble` is a prefix parser, so a leading numeric prefix is -/// enough for `YAML.parse` to resolve a number — e.g. `"1+5"` round-trips to `1`. -/// We err on the side of quoting when the parser's scanner would accept the full token -/// as `valid`. +/// This mirrors the rules in `src/parsers/yaml.rs`'s `try_resolve_number` / +/// `is_core_schema_number`: +/// - Optional leading sign, optionally followed by `.inf`/`.Inf`/`.INF`. +/// - Otherwise either an integer (`[0-9]+` / `0x…` / `0o…`) or a float +/// matching §10.2.1.4 `[-+]? ( . [0-9]+ | [0-9]+ ( . [0-9]* )? ) ([eE][-+]?[0-9]+)?`. +/// The parser-side gate now rejects non-conforming float-like tokens +/// (e.g. `"1+5"`, `"1e"`, `"."`) so this mirror should err on the side of +/// *quoting* whenever a token *might* parse as a number. fn string_is_number(str: &BunString) -> bool { let len = str.length(); if len == 0 { diff --git a/test/js/bun/yaml/yaml.test.ts b/test/js/bun/yaml/yaml.test.ts index 7ef7179890c..7dda7b23392 100644 --- a/test/js/bun/yaml/yaml.test.ts +++ b/test/js/bun/yaml/yaml.test.ts @@ -1216,10 +1216,7 @@ folded: > test("two anchors before e-node `:` (outer=mapping, inner=key)", () => { // Valid per [200]/[193]: outer anchors the collection, inner the - // e-node key. The Scalar-key analogue (`&outer\n&inner b: x`) is - // accepted because that arm `return Ok(mapping)`, bypassing the - // post-loop has_mapping_anchor guard; the e-node arm reaches it. - // Pre-existing on main. + // e-node key. expect(YAML.parse("&outer\n&inner : x\n")).toEqual({ null: "x" }); }); @@ -1782,7 +1779,7 @@ folded: > ["-.e5", "-.e5"], ["e5", "e5"], ] as const)("%s resolves as string", (input, expected) => { - expect(Bun.YAML.parse(input)).toBe(expected); + expect(YAML.parse(input)).toBe(expected); }); test.each([ ["1", 1], @@ -1803,10 +1800,18 @@ folded: > ["-.5e2", -50], ["0x1f", 31], ["0o17", 15], - ["-0x1f", -31], ] as const)("%s resolves as number %p", (input, expected) => { - expect(Bun.YAML.parse(input)).toBe(expected); - }); + expect(YAML.parse(input)).toBe(expected); + }); + test.todo.each(["-0x1f", "+0x1f", "-0o17", "+0o17"])( + "signed hex/octal %s resolves as string (§10.2.1.2)", + input => { + // Core schema int regex is `0x [0-9a-fA-F]+` — no sign. js-yaml, + // PyYAML, ruamel agree. Pre-existing on the int path (not gated + // by is_core_schema_number, which only validates the float path). + expect(YAML.parse(input)).toBe(input); + }, + ); test.each([ [".inf", Infinity], ["+.inf", Infinity], @@ -1816,12 +1821,12 @@ folded: > ["-.Inf", -Infinity], ["-.INF", -Infinity], ] as const)("%s resolves as %p", (input, expected) => { - expect(Bun.YAML.parse(input)).toBe(expected); + expect(YAML.parse(input)).toBe(expected); }); test(".nan resolves as NaN", () => { - expect(Bun.YAML.parse(".nan")).toBeNaN(); - expect(Bun.YAML.parse(".NaN")).toBeNaN(); - expect(Bun.YAML.parse(".NAN")).toBeNaN(); + expect(YAML.parse(".nan")).toBeNaN(); + expect(YAML.parse(".NaN")).toBeNaN(); + expect(YAML.parse(".NAN")).toBeNaN(); }); });