Skip to content
6 changes: 3 additions & 3 deletions src/v2/compiler/02_parse.dag
Original file line number Diff line number Diff line change
Expand Up @@ -1762,18 +1762,18 @@ fn grammar_validate_for_parse(root: GrammarRoot, at: Locus) -> Outcome<Validated
}
}

fn parse_sync_step(acc: ParseSyncState, tok: Token, sync_tokens: Set<Symbol>) -> ParseSyncState {
fn parse_sync_step(acc: ParseSyncState, tok: Token, sync_tokens: List<Symbol>) -> ParseSyncState {
match acc.found_sync {
true => ParseSyncState { found_sync: true, result: list_snoc_item(xs: acc.result, item: tok) }
false =>
match sync_tokens.member(tok.class) {
match contains(xs: sync_tokens, item: tok.class, eq: fn(a, b) { a == b }) {
true => ParseSyncState { found_sync: true, result: list_snoc_item(xs: std.algebra.Empty, item: tok) }
false => ParseSyncState { found_sync: false, result: std.algebra.Empty }
}
}
}

fn parse_skip_to_sync(tokens: List<Token>, sync_tokens: Set<Symbol>) -> ParseSyncState {
fn parse_skip_to_sync(tokens: List<Token>, sync_tokens: List<Symbol>) -> ParseSyncState {
fold_list(
xs: tokens,
empty: ParseSyncState { found_sync: false, result: std.algebra.Empty },
Expand Down
20 changes: 9 additions & 11 deletions src/v2/extdeps/languages/dag.dag
Original file line number Diff line number Diff line change
Expand Up @@ -3456,17 +3456,15 @@ fn dag_lex_token_symbol(sym: Symbol) -> Bool {
)
}

fn dag_sync_tokens() -> Set<Symbol> {
Set {
member: fn(sym) {
sym == ^dag_token_kw_module ||
sym == ^dag_token_kw_import ||
sym == ^dag_token_kw_alias ||
sym == ^dag_token_kw_type ||
sym == ^dag_token_kw_data ||
sym == ^dag_token_kw_fn
}
}
fn dag_sync_tokens() -> List<Symbol> {
[
^dag_token_kw_module,
^dag_token_kw_import,
^dag_token_kw_alias,
^dag_token_kw_type,
^dag_token_kw_data,
^dag_token_kw_fn
]
}


Expand Down
23 changes: 20 additions & 3 deletions src/v2/std/grammar.dag
Original file line number Diff line number Diff line change
Expand Up @@ -289,10 +289,27 @@ type GrammarProduction {
emitted: Node
}

// SYNC TOKENS ARE AN ENUMERATION, NOT A PREDICATE.
//
// A grammar's error-recovery sync tokens are finite and known at authoring time — every producer in
// the corpus writes either an empty set or a literal list of keyword classes. Carrying them as
// `Set<Symbol>` made them a POINTWISE-POWER characteristic function (`std.types` grounds
// `Set<element> = PointwisePower<element>`), which is the right grounding for a set in general and
// the wrong carrier for six literal keywords: a characteristic function cannot be enumerated, so
// the whole grammar value became unhashable through this one field.
//
// That was not a theoretical loss. `prepare_grammar` is memoized cross-claim on
// (declaration identity, content hash of the grammar), and the interpreter's content-hash walk
// BAILS on a closure — so the memo could never form its key, never store, and never hit, for every
// language in the corpus and not only `.dag`. The measured consequence was a full ~12s grammar
// preparation on every parse, including of an empty module.
//
// The field is therefore the enumeration it always was. `Set` keeps its grounding untouched; what
// changed is that this field stopped borrowing a power-set to carry a list.
type GrammarRoot {
start: Symbol
productions: List<GrammarProduction>
sync_tokens: Set<Symbol>
sync_tokens: List<Symbol>
}

type ValidatedGrammarRoot {
Expand Down Expand Up @@ -347,8 +364,8 @@ fn grammar_named_edge(name: Symbol, target: Node) -> Edge {
}
}

fn grammar_empty_sync_tokens() -> Set<Symbol> {
Set { member: fn(_) { false } }
fn grammar_empty_sync_tokens() -> List<Symbol> {
[]
}

fn grammar_empty_node() -> Node {
Expand Down