Skip to content

v2 parse: layout derived from the stream's source; RefuseOnMatch and AfterLineBreak grammar arms (PR1 of 2) #12773

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gunbai-bot merged 17 commits into from
Oct 1, 2026
Merged

v2 parse: layout derived from the stream's source; RefuseOnMatch and AfterLineBreak grammar arms (PR1 of 2) #12773

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
8afdb1e
WIP layout model: TokenStream source, named line-break derivation, Re…
Sep 30, 2026
8371ea0
Layout control: same tokens, different layout, different stream digests
Sep 30, 2026
ef567ab
Layout: derive a per-token line-break index once per stream; a layout…
Sep 30, 2026
22131ce
Layout identity control parses against the warmed dag_prepared_gramma…
Sep 30, 2026
24b7716
Layout identity control: a module that reaches no layout question (+,…
Sep 30, 2026
0f04f7f
Layout: keep only line-break-preceded starts; digest covers layout by…
Sep 30, 2026
dcc31bb
grammar: cite token_stream_line_break_before (stale symbol, review 73…
Sep 30, 2026
a30b62f
lexing: delete token_stream_source, no consumer after the layout rewo…
Sep 30, 2026
e4e899b
StreamLayout: drop the unread source field (the source is read once, …
Sep 30, 2026
39bbbd4
layout test: the digest covers layout, not source (comment)
Sep 30, 2026
66b9ff1
lexing: spell the tokenizer's source as v2.std.text.String (emit-buil…
Sep 30, 2026
f7888e6
StreamLayout: one representation, start -> line of each line-break-pr…
Sep 30, 2026
cf486fd
Merge remote-tracking branch 'origin/main' into session/calm-fox-43-l…
Oct 1, 2026
26ad3f4
Layout digest folded once in the layout walk; the parse table reads i…
Oct 1, 2026
6094f43
lexing: type the layout digest as Fnv1a64Structural, what combine_has…
Oct 1, 2026
def354a
Layout recorded in the tokenizer's own walk (no second pass over the …
Oct 1, 2026
7a73163
lexing: lexeme_has_line_feed takes v2.std.text.String (emit)
Oct 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 45 additions & 11 deletions src/v2/compiler/01_tokenize.dag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@ import v2.std.compilers.lexing {
lex_artifact_new,
lex_artifact_semantic_tokens,
token_stream_empty,
token_stream_new,
lexeme_has_line_feed,
stream_layout_empty,
stream_layout_with_break,
StreamLayout,
token_stream_lexed,
TriviaRule,
UnboundSourceAnnotation,
VoidLexRules,
Expand Down Expand Up @@ -659,6 +663,9 @@ type LexWalkAcc {
error: Bool
file: Symbol
dispatch: LexRuleDispatch
gap_start: Int
gap_has_break: Bool
layout: StreamLayout
}

fn lex_walk_init(src: v2.std.text.String, file: Symbol, rules: LexRuleSet) -> LexWalkAcc {
Expand All @@ -671,7 +678,10 @@ fn lex_walk_init(src: v2.std.text.String, file: Symbol, rules: LexRuleSet) -> Le
done: false,
error: false,
file: file,
dispatch: lex_rule_dispatch(compiled_rules: lex_compile_rules(rules: rules.rules))
dispatch: lex_rule_dispatch(compiled_rules: lex_compile_rules(rules: rules.rules)),
gap_start: 0,
gap_has_break: false,
layout: stream_layout_empty()
}
}

Expand All @@ -691,7 +701,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: true,
error: false,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break,
layout: acc.layout
}
false =>
match lex_try_rules(rules: lex_dispatch_candidates(dispatch: acc.dispatch, source: acc.remaining), source: acc.remaining) {
Expand All @@ -705,7 +718,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: true,
error: true,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break,
layout: acc.layout
}
LexRuleTrivia { lexeme: lexeme, remaining: remaining } =>
match string_is_empty(s: lexeme) {
Expand All @@ -719,7 +735,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: true,
error: true,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break,
layout: acc.layout
}
false =>
LexWalkAcc {
Expand All @@ -731,7 +750,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: false,
error: false,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break || lexeme_has_line_feed(lexeme: lexeme),
layout: acc.layout
}
}
LexRuleToken { class: class, lexeme: lexeme, remaining: remaining } =>
Expand All @@ -746,7 +768,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: true,
error: true,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break,
layout: acc.layout
}
false =>
LexWalkAcc {
Expand All @@ -767,7 +792,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: false,
error: false,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.pos + lexeme_utf8_octet_count(lexeme: lexeme),
gap_has_break: false,
layout: if acc.gap_has_break { stream_layout_with_break(layout: acc.layout, token_start: acc.pos, gap_start: acc.gap_start) } else { acc.layout }
}
}
LexRuleAnnotation { class: class, lexeme: lexeme, remaining: remaining } =>
Expand All @@ -782,7 +810,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: true,
error: true,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break,
layout: acc.layout
}
false =>
LexWalkAcc {
Expand All @@ -804,7 +835,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc {
done: false,
error: false,
file: acc.file,
dispatch: acc.dispatch
dispatch: acc.dispatch,
gap_start: acc.gap_start,
gap_has_break: acc.gap_has_break || lexeme_has_line_feed(lexeme: lexeme),
layout: acc.layout
}
}
}
Expand Down Expand Up @@ -843,7 +877,7 @@ fn lex_walk_artifact(source: v2.std.text.String, file: Symbol, rules: LexRuleSet
} else {
v2.std.diagnostic.Accepted {
value: lex_artifact_new(
tokens: token_stream_new(all: result.tokens.reverse()),
tokens: token_stream_lexed(all: result.tokens.reverse(), layout: result.layout),
annotations: result.annotations.reverse()
),
diagnostics: v2.std.diagnostic.None
Expand Down
Loading