diff --git a/src/v2/compiler/target_serialize.dag b/src/v2/compiler/target_serialize.dag index e405ef15c5d..f04e4df627d 100644 --- a/src/v2/compiler/target_serialize.dag +++ b/src/v2/compiler/target_serialize.dag @@ -102,7 +102,7 @@ import v2.std.compilers.target_model { target_text_empty, target_text_is_empty, render_target_text, - token_text_from_model, + token_text_from_decoded, target_text_nested, concrete_syntax_token_from_node, target_binding_spelling_lookup, @@ -201,7 +201,7 @@ fn translate_serialize_measure(target: TargetModel, emitted: Node) -> Outcome } @@ -535,8 +535,8 @@ fn token_sequence_item_kind_flat(item: Node) -> Outcome { bind_outcome( o: concrete_syntax_token_from_node(token: item), - f: fn(_) { - Accepted { value: TokenSequenceConcreteToken, diagnostics: None } + f: fn(decoded) { + Accepted { value: TokenSequenceConcreteToken { token: decoded }, diagnostics: None } } ) Absent => @@ -960,8 +960,8 @@ fn token_item_to_source_bounded( o: token_sequence_item_kind(item: item), f: fn(item_kind) { match item_kind { - TokenSequenceConcreteToken => - token_text_from_model(target: target, token: item) + TokenSequenceConcreteToken { token: decoded } => + token_text_from_decoded(target: target, decoded: decoded) TokenSequenceNestedBlock { unit: unit, tokens: inner } => serialize_relation_row_nested_bounded(remaining - 1, target, unit, inner) TokenSequenceNonterminal => diff --git a/src/v2/std/compilers/target_model.dag b/src/v2/std/compilers/target_model.dag index 6813ce8fc1b..50a92be069d 100644 --- a/src/v2/std/compilers/target_model.dag +++ b/src/v2/std/compilers/target_model.dag @@ -1050,7 +1050,7 @@ fn apply_emit_spelling_escapes(spelling: v2.std.text.String, escapes: List Diagnostic { Diagnostic { reason: ^target_layout_token_spelled_flat, @@ -1156,25 +1156,22 @@ fn layout_or_atom(target: TargetModel, token_class: Symbol, spelling: String) -> // THE CARRIER-NATIVE TOKEN SPELLING: the lexeme (fixed: the lex literal; bound: the binding's // spelling) under the token class's emit transform, as TargetText, so an EmitLineBreaks class keeps -// its line breaks. -fn token_text_from_model(target: TargetModel, token: Node) -> Outcome { - bind_outcome( - o: concrete_syntax_token_from_node(token: token), - f: fn(decoded) { - match decoded { - FixedToken { token_class: token_class } => - bind_outcome( - o: lex_rules_literal_for_class(rules: target.lex, token_class: token_class), - f: fn(spelling) { layout_or_atom(target: target, token_class: token_class, spelling: spelling) } - ) - BoundToken { token_class: token_class, binding: binding } => - bind_outcome( - o: target_binding_spelling_lookup(target: target, binding: binding), - f: fn(spelling) { layout_or_atom(target: target, token_class: token_class, spelling: spelling) } - ) - } - } - ) +// its line breaks. It takes the token already decoded: the serializer decodes each token node once to +// classify it (v2.compiler.target_serialize token_sequence_item_kind) and hands that value here, so a +// token is not decoded a second time to be spelled. +fn token_text_from_decoded(target: TargetModel, decoded: ConcreteSyntaxToken) -> Outcome { + match decoded { + FixedToken { token_class: token_class } => + bind_outcome( + o: lex_rules_literal_for_class(rules: target.lex, token_class: token_class), + f: fn(spelling) { layout_or_atom(target: target, token_class: token_class, spelling: spelling) } + ) + BoundToken { token_class: token_class, binding: binding } => + bind_outcome( + o: target_binding_spelling_lookup(target: target, binding: binding), + f: fn(spelling) { layout_or_atom(target: target, token_class: token_class, spelling: spelling) } + ) + } } fn token_spelling_from_model(target: TargetModel, token: Node) -> Outcome {