"); + rewriter.insertBefore(0, ""); + rewriter.insertAfter(0, "
"); + rewriter.insertAfter(0, ""); + rewriter.insertBefore(1, ""); + rewriter.insertAfter(1, ""); + + // Assert + expect(rewriter.getText()).toEqual("a
a"); + }); + + it("preserves the order of contiguous inserts", () => { + // Arrange + const rewriter = getRewriter(abc, "ab"); + + // Act + rewriter.insertBefore(0, "");
+ rewriter.insertBefore(0, "");
+ rewriter.insertBefore(0, " a
+ * This token stream ignores the value of {@link Token//getChannel}. If your + * parser requires the token stream filter tokens to only those on a particular + * channel, such as {@link Token//DEFAULT_CHANNEL} or + * {@link Token//HIDDEN_CHANNEL}, use a filtering token stream such a + * {@link CommonTokenStream}.
+ */ +export default class BufferedTokenStream extends TokenStream { + constructor(tokenSource) { + + super(); + // The {@link TokenSource} from which tokens for this stream are fetched. + this.tokenSource = tokenSource; + /** + * A collection of all tokens fetched from the token source. The list is + * considered a complete view of the input once {@link //fetchedEOF} is set + * to {@code true}. + */ + this.tokens = []; + + /** + * The index into {@link //tokens} of the current token (next token to + * {@link //consume}). {@link //tokens}{@code [}{@link //p}{@code ]} should + * be + * {@link //LT LT(1)}. + * + *This field is set to -1 when the stream is first constructed or when + * {@link //setTokenSource} is called, indicating that the first token has + * not yet been fetched from the token source. For additional information, + * see the documentation of {@link IntStream} for a description of + * Initializing Methods.
+ */ + this.index = -1; + + /** + * Indicates whether the {@link Token//EOF} token has been fetched from + * {@link //tokenSource} and added to {@link //tokens}. This field improves + * performance for the following cases: + * + *For example, {@link CommonTokenStream} overrides this method to ensure + * that + * the seek target is always an on-channel token.
+ * + * @param {Number} i The target token index. + * @return {Number} The adjusted target token index. + */ + adjustSeekIndex(i) { + return i; + } + + lazyInit() { + if (this.index === -1) { + this.setup(); + } + } + + setup() { + this.sync(0); + this.index = this.adjustSeekIndex(0); + } + + // Reset this token stream by setting its token source./// + setTokenSource(tokenSource) { + this.tokenSource = tokenSource; + this.tokens = []; + this.index = -1; + this.fetchedEOF = false; + } + + /** + * Given a starting index, return the index of the next token on channel. + * Return i if tokens[i] is on channel. Return -1 if there are no tokens + * on channel between i and EOF. + */ + nextTokenOnChannel(i, channel) { + this.sync(i); + if (i >= this.tokens.length) { + return -1; + } + let token = this.tokens[i]; + while (token.channel !== this.channel) { + if (token.type === Token.EOF) { + return -1; + } + i += 1; + this.sync(i); + token = this.tokens[i]; + } + return i; + } + + /** + * Given a starting index, return the index of the previous token on channel. + * Return i if tokens[i] is on channel. Return -1 if there are no tokens + * on channel between i and 0. + */ + previousTokenOnChannel(i, channel) { + while (i >= 0 && this.tokens[i].channel !== channel) { + i -= 1; + } + return i; + } + + /** + * Collect all tokens on specified channel to the right of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or + * EOF. If channel is -1, find any non default channel token. + */ + getHiddenTokensToRight(tokenIndex, + channel) { + if (channel === undefined) { + channel = -1; + } + this.lazyInit(); + if (tokenIndex < 0 || tokenIndex >= this.tokens.length) { + throw "" + tokenIndex + " not in 0.." + this.tokens.length - 1; + } + const nextOnChannel = this.nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL); + const from_ = tokenIndex + 1; + // if none onchannel to right, nextOnChannel=-1 so set to = last token + const to = nextOnChannel === -1 ? this.tokens.length - 1 : nextOnChannel; + return this.filterForChannel(from_, to, channel); + } + + /** + * Collect all tokens on specified channel to the left of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + * If channel is -1, find any non default channel token. + */ + getHiddenTokensToLeft(tokenIndex, + channel) { + if (channel === undefined) { + channel = -1; + } + this.lazyInit(); + if (tokenIndex < 0 || tokenIndex >= this.tokens.length) { + throw "" + tokenIndex + " not in 0.." + this.tokens.length - 1; + } + const prevOnChannel = this.previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL); + if (prevOnChannel === tokenIndex - 1) { + return null; + } + // if none on channel to left, prevOnChannel=-1 then from=0 + const from_ = prevOnChannel + 1; + const to = tokenIndex - 1; + return this.filterForChannel(from_, to, channel); + } + + filterForChannel(left, right, channel) { + const hidden = []; + for (let i = left; i < right + 1; i++) { + const t = this.tokens[i]; + if (channel === -1) { + if (t.channel !== Lexer.DEFAULT_TOKEN_CHANNEL) { + hidden.push(t); + } + } else if (t.channel === channel) { + hidden.push(t); + } + } + if (hidden.length === 0) { + return null; + } + return hidden; + } + + getSourceName() { + return this.tokenSource.getSourceName(); + } + + // Get the text of all tokens in this buffer./// + getText(interval) { + this.lazyInit(); + this.fill(); + if (!interval) { + interval = new Interval(0, this.tokens.length - 1); + } + let start = interval.start; + if (start instanceof Token) { + start = start.tokenIndex; + } + let stop = interval.stop; + if (stop instanceof Token) { + stop = stop.tokenIndex; + } + if (start === null || stop === null || start < 0 || stop < 0) { + return ""; + } + if (stop >= this.tokens.length) { + stop = this.tokens.length - 1; + } + let s = ""; + for (let i = start; i < stop + 1; i++) { + const t = this.tokens[i]; + if (t.type === Token.EOF) { + break; + } + s = s + t.text; + } + return s; + } + + // Get all tokens from lexer until EOF/// + fill() { + this.lazyInit(); + // noinspection StatementWithEmptyBodyJS + while (this.fetch(1000) === 1000); + } +} + +Object.defineProperty(BufferedTokenStream, "size", { + get: function() { + return this.tokens.length; + } +}) diff --git a/packages/antlr4/src/antlr4/CharStream.d.ts b/packages/antlr4/src/antlr4/CharStream.d.ts new file mode 100644 index 000000000..2a2e06eed --- /dev/null +++ b/packages/antlr4/src/antlr4/CharStream.d.ts @@ -0,0 +1,17 @@ +export declare class CharStream { + + index: number; // defined as property + size: number;// defined as property + + constructor(data: string); + constructor(data: string, decodeToUnicodeCodePoints: boolean); + reset(): void; + consume(): void; + LA(offset: number): number; + LT(offset: number): number; + mark(): number; + release(marker: number): void; + seek(index: number): void; + getText(start: number, stop: number): string; + toString(): string; +} diff --git a/packages/antlr4/src/antlr4/CharStream.js b/packages/antlr4/src/antlr4/CharStream.js new file mode 100644 index 000000000..718aa2d96 --- /dev/null +++ b/packages/antlr4/src/antlr4/CharStream.js @@ -0,0 +1,125 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import Token from './Token.js'; + +/** + * If decodeToUnicodeCodePoints is true, the input is treated + * as a series of Unicode code points. + * + * Otherwise, the input is treated as a series of 16-bit UTF-16 code + * units. + */ +export default class CharStream { + constructor(data, decodeToUnicodeCodePoints) { + this.name = "+ * If {@code oldToken} is also a {@link CommonToken} instance, the newly + * constructed token will share a reference to the {@link //text} field and + * the {@link Pair} stored in {@link //source}. Otherwise, {@link //text} will + * be assigned the result of calling {@link //getText}, and {@link //source} + * will be constructed from the result of {@link Token//getTokenSource} and + * {@link Token//getInputStream}.
+ * + * @param oldToken The token to copy. + */ + clone() { + const t = new CommonToken(this.source, this.type, this.channel, this.start, this.stop); + t.tokenIndex = this.tokenIndex; + t.line = this.line; + t.column = this.column; + t.text = this.text; + return t; + } + + cloneWithType(type) { + const t = new CommonToken(this.source, type, this.channel, this.start, this.stop); + t.tokenIndex = this.tokenIndex; + t.line = this.line; + t.column = this.column; + if (type === Token.EOF) + t.text = ""; + return t; + } + + toString() { + let txt = this.text; + if (txt !== null) { + txt = txt.replace(/\n/g, "\\n").replace(/\r/g, "\\r").replace(/\t/g, "\\t"); + } else { + txt = "+ * The default value is {@code false} to avoid the performance and memory + * overhead of copying text for every token unless explicitly requested.
+ */ + this.copyText = copyText===undefined ? false : copyText; + } + + create(source, type, text, channel, start, stop, line, column) { + const t = new CommonToken(source, type, channel, start, stop); + t.line = line; + t.column = column; + if (text !==null) { + t.text = text; + } else if (this.copyText && source[1] !==null) { + t.text = source[1].getText(start,stop); + } + return t; + } + + createThin(type, text) { + const t = new CommonToken(null, type); + t.text = text; + return t; + } +} + +/** + * The default {@link CommonTokenFactory} instance. + * + *+ * This token factory does not explicitly copy token text when constructing + * tokens.
+ */ +CommonTokenFactory.DEFAULT = new CommonTokenFactory(); diff --git a/packages/antlr4/src/antlr4/CommonTokenStream.d.ts b/packages/antlr4/src/antlr4/CommonTokenStream.d.ts new file mode 100644 index 000000000..92ede4f71 --- /dev/null +++ b/packages/antlr4/src/antlr4/CommonTokenStream.d.ts @@ -0,0 +1,11 @@ +import {Lexer} from "./Lexer"; +import {BufferedTokenStream} from "./BufferedTokenStream"; + +export declare class CommonTokenStream extends BufferedTokenStream { + // properties + tokens: string[]; + // methods + constructor(lexer: Lexer); + constructor(lexer: Lexer, channel: number); + fill(): void; +} diff --git a/packages/antlr4/src/antlr4/CommonTokenStream.js b/packages/antlr4/src/antlr4/CommonTokenStream.js new file mode 100644 index 000000000..0e44e009a --- /dev/null +++ b/packages/antlr4/src/antlr4/CommonTokenStream.js @@ -0,0 +1,98 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + + +import Token from './Token.js'; +import BufferedTokenStream from './BufferedTokenStream.js'; + +/** + * This class extends {@link BufferedTokenStream} with functionality to filter + * token streams to tokens on a particular channel (tokens where + * {@link Token//getChannel} returns a particular value). + * + *+ * This token stream provides access to all tokens by index or when calling + * methods like {@link //getText}. The channel filtering is only used for code + * accessing tokens via the lookahead methods {@link //LA}, {@link //LT}, and + * {@link //LB}.
+ * + *+ * By default, tokens are placed on the default channel + * ({@link Token//DEFAULT_CHANNEL}), but may be reassigned by using the + * {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to + * call {@link Lexer//setChannel}. + *
+ * + *+ * Note: lexer rules which use the {@code ->skip} lexer command or call + * {@link Lexer//skip} do not produce tokens at all, so input text matched by + * such a rule will not be available as part of the token stream, regardless of + * channel.
+ */ +export default class CommonTokenStream extends BufferedTokenStream { + constructor(lexer, channel) { + super(lexer); + this.channel = channel===undefined ? Token.DEFAULT_CHANNEL : channel; + } + + adjustSeekIndex(i) { + return this.nextTokenOnChannel(i, this.channel); + } + + LB(k) { + if (k===0 || this.index-k<0) { + return null; + } + let i = this.index; + let n = 1; + // find k good tokens looking backwards + while (n <= k) { + // skip off-channel tokens + i = this.previousTokenOnChannel(i - 1, this.channel); + n += 1; + } + if (i < 0) { + return null; + } + return this.tokens[i]; + } + + LT(k) { + this.lazyInit(); + if (k === 0) { + return null; + } + if (k < 0) { + return this.LB(-k); + } + let i = this.index; + let n = 1; // we know tokens[pos] is a good one + // find k good tokens + while (n < k) { + // skip off-channel tokens, but make sure to not look past EOF + if (this.sync(i + 1)) { + i = this.nextTokenOnChannel(i + 1, this.channel); + } + n += 1; + } + return this.tokens[i]; + } + + // Count EOF just once. + getNumberOfOnChannelTokens() { + let n = 0; + this.fill(); + for (let i =0; i< this.tokens.length;i++) { + const t = this.tokens[i]; + if( t.channel===this.channel) { + n += 1; + } + if( t.type===Token.EOF) { + break; + } + } + return n; + } +} diff --git a/packages/antlr4/src/antlr4/FileStream.d.ts b/packages/antlr4/src/antlr4/FileStream.d.ts new file mode 100644 index 000000000..204b09dcb --- /dev/null +++ b/packages/antlr4/src/antlr4/FileStream.d.ts @@ -0,0 +1,9 @@ +import {CharStream} from "./CharStream"; + +export declare class FileStream extends CharStream { + + fileName: string; + + constructor(fileName: string, encoding?: string, decodeToUnicodeCodePoints?: boolean); + +} diff --git a/packages/antlr4/src/antlr4/FileStream.js b/packages/antlr4/src/antlr4/FileStream.js new file mode 100644 index 000000000..fce1d33d8 --- /dev/null +++ b/packages/antlr4/src/antlr4/FileStream.js @@ -0,0 +1,40 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import InputStream from './InputStream.js'; +import CharStream from './CharStream.js'; +const isNode = + typeof process !== "undefined" && + process.versions != null && + process.versions.node != null; +import fs from 'fs'; + +/** + * This is an InputStream that is loaded from a file all at once + * when you construct the object. + */ +export default class FileStream extends InputStream { + + static fromPath(path, encoding, callback) { + if(!isNode) + throw new Error("FileStream is only available when running in Node!"); + fs.readFile(path, encoding, function(err, data) { + let is = null; + if (data !== null) { + is = new CharStream(data, true); + } + callback(err, is); + }); + + } + + constructor(fileName, encoding, decodeToUnicodeCodePoints) { + if(!isNode) + throw new Error("FileStream is only available when running in Node!"); + const data = fs.readFileSync(fileName, encoding || "utf-8" ); + super(data, decodeToUnicodeCodePoints); + this.fileName = fileName; + } +} diff --git a/packages/antlr4/src/antlr4/InputStream.d.ts b/packages/antlr4/src/antlr4/InputStream.d.ts new file mode 100644 index 000000000..fa341d7ca --- /dev/null +++ b/packages/antlr4/src/antlr4/InputStream.d.ts @@ -0,0 +1,4 @@ +export declare class InputStream { + constructor(data: string); + constructor(data: string, decodeToUnicodeCodePoints: boolean); +} diff --git a/packages/antlr4/src/antlr4/InputStream.js b/packages/antlr4/src/antlr4/InputStream.js new file mode 100644 index 000000000..943c0669e --- /dev/null +++ b/packages/antlr4/src/antlr4/InputStream.js @@ -0,0 +1,15 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import CharStream from './CharStream.js'; + +/** + * @deprecated Use CharStream instead +*/ +export default class InputStream extends CharStream { + constructor(data, decodeToUnicodeCodePoints) { + super(data, decodeToUnicodeCodePoints); + } +} diff --git a/packages/antlr4/src/antlr4/Lexer.d.ts b/packages/antlr4/src/antlr4/Lexer.d.ts new file mode 100644 index 000000000..fe32aa7b4 --- /dev/null +++ b/packages/antlr4/src/antlr4/Lexer.d.ts @@ -0,0 +1,32 @@ +import {Recognizer} from "./Recognizer"; +import {LexerATNSimulator} from "./atn"; +import {CharStream} from "./CharStream"; +import {Token} from "./Token"; + +export declare class Lexer extends RecognizerIf the symbol type does not match, + * {@link ANTLRErrorStrategy//recoverInline} is called on the current error + * strategy to attempt recovery. If {@link //buildParseTree} is + * {@code true} and the token index of the symbol returned by + * {@link ANTLRErrorStrategy//recoverInline} is -1, the symbol is added to + * the parse tree by calling {@link ParserRuleContext//addErrorNode}.
+ * + * @param ttype the token type to match + * @return the matched symbol + * @throws RecognitionException if the current input symbol did not match + * {@code ttype} and the error strategy could not recover from the + * mismatched symbol + */ + match(ttype) { + let t = this.getCurrentToken(); + if (t.type === ttype) { + this._errHandler.reportMatch(this); + this.consume(); + } else { + t = this._errHandler.recoverInline(this); + if (this.buildParseTrees && t.tokenIndex === -1) { + // we must have conjured up a new token during single token + // insertion + // if it's not the current symbol + this._ctx.addErrorNode(t); + } + } + return t; + } + + /** + * Match current input symbol as a wildcard. If the symbol type matches + * (i.e. has a value greater than 0), {@link ANTLRErrorStrategy//reportMatch} + * and {@link //consume} are called to complete the match process. + * + *If the symbol type does not match, + * {@link ANTLRErrorStrategy//recoverInline} is called on the current error + * strategy to attempt recovery. If {@link //buildParseTree} is + * {@code true} and the token index of the symbol returned by + * {@link ANTLRErrorStrategy//recoverInline} is -1, the symbol is added to + * the parse tree by calling {@link ParserRuleContext//addErrorNode}.
+ * + * @return the matched symbol + * @throws RecognitionException if the current input symbol did not match + * a wildcard and the error strategy could not recover from the mismatched + * symbol + */ + matchWildcard() { + let t = this.getCurrentToken(); + if (t.type > 0) { + this._errHandler.reportMatch(this); + this.consume(); + } else { + t = this._errHandler.recoverInline(this); + if (this.buildParseTrees && t.tokenIndex === -1) { + // we must have conjured up a new token during single token + // insertion + // if it's not the current symbol + this._ctx.addErrorNode(t); + } + } + return t; + } + + getParseListeners() { + return this._parseListeners || []; + } + + /** + * Registers {@code listener} to receive events during the parsing process. + * + *To support output-preserving grammar transformations (including but not + * limited to left-recursion removal, automated left-factoring, and + * optimized code generation), calls to listener methods during the parse + * may differ substantially from calls made by + * {@link ParseTreeWalker//DEFAULT} used after the parse is complete. In + * particular, rule entry and exit events may occur in a different order + * during the parse than after the parser. In addition, calls to certain + * rule entry methods may be omitted.
+ * + *With the following specific exceptions, calls to listener events are + * deterministic, i.e. for identical input the calls to listener + * methods will be the same.
+ * + *If {@code listener} is {@code null} or has not been added as a parse + * listener, this method does nothing.
+ * @param listener the listener to remove + */ + removeParseListener(listener) { + if (this._parseListeners !== null) { + const idx = this._parseListeners.indexOf(listener); + if (idx >= 0) { + this._parseListeners.splice(idx, 1); + } + if (this._parseListeners.length === 0) { + this._parseListeners = null; + } + } + } + + // Remove all parse listeners. + removeParseListeners() { + this._parseListeners = null; + } + + // Notify any parse listeners of an enter rule event. + triggerEnterRuleEvent() { + if (this._parseListeners !== null) { + const ctx = this._ctx; + this._parseListeners.forEach(function (listener) { + listener.enterEveryRule(ctx); + ctx.enterRule(listener); + }); + } + } + + /** + * Notify any parse listeners of an exit rule event. + * @see //addParseListener + */ + triggerExitRuleEvent() { + if (this._parseListeners !== null) { + // reverse order walk of listeners + const ctx = this._ctx; + this._parseListeners.slice(0).reverse().forEach(function (listener) { + ctx.exitRule(listener); + listener.exitEveryRule(ctx); + }); + } + } + + getTokenFactory() { + return this._input.tokenSource._factory; + } + + // Tell our token source and error strategy about a new way to create tokens. + setTokenFactory(factory) { + this._input.tokenSource._factory = factory; + } + + /** + * The ATN with bypass alternatives is expensive to create so we create it + * lazily. + * + * @throws UnsupportedOperationException if the current parser does not + * implement the {@link //getSerializedATN()} method. + */ + getATNWithBypassAlts() { + const serializedAtn = this.getSerializedATN(); + if (serializedAtn === null) { + throw "The current parser does not support an ATN with bypass alternatives."; + } + let result = this.bypassAltsAtnCache[serializedAtn]; + if (result === null) { + const deserializationOptions = new ATNDeserializationOptions(); + deserializationOptions.generateRuleBypassTransitions = true; + result = new ATNDeserializer(deserializationOptions) + .deserialize(serializedAtn); + this.bypassAltsAtnCache[serializedAtn] = result; + } + return result; + } + + getInputStream() { + return this.getTokenStream(); + } + + setInputStream(input) { + this.setTokenStream(input); + } + + getTokenStream() { + return this._input; + } + + // Set the token stream and reset the parser. + setTokenStream(input) { + this._input = null; + this.reset(); + this._input = input; + } + + /** + * Gets the number of syntax errors reported during parsing. This value is + * incremented each time {@link //notifyErrorListeners} is called. + */ + get syntaxErrorsCount() { + return this._syntaxErrors; + } + + + /** + * Match needs to return the current input symbol, which gets put + * into the label for the associated token ref; e.g., x=ID. + */ + getCurrentToken() { + return this._input.LT(1); + } + + notifyErrorListeners(msg, offendingToken, err) { + offendingToken = offendingToken || null; + err = err || null; + if (offendingToken === null) { + offendingToken = this.getCurrentToken(); + } + this._syntaxErrors += 1; + const line = offendingToken.line; + const column = offendingToken.column; + const listener = this.getErrorListenerDispatch(); + listener.syntaxError(this, offendingToken, line, column, msg, err); + } + + /** + * Consume and return the {@linkplain //getCurrentToken current symbol}. + * + *E.g., given the following input with {@code A} being the current + * lookahead symbol, this function moves the cursor to {@code B} and returns + * {@code A}.
+ * + *+ * A B + * ^ + *+ * + * If the parser is not in error recovery mode, the consumed symbol is added + * to the parse tree using {@link ParserRuleContext//addChild(Token)}, and + * {@link ParseTreeListener//visitTerminal} is called on any parse listeners. + * If the parser is in error recovery mode, the consumed symbol is + * added to the parse tree using + * {@link ParserRuleContext//addErrorNode(Token)}, and + * {@link ParseTreeListener//visitErrorNode} is called on any parse + * listeners. + */ + consume() { + const o = this.getCurrentToken(); + if (o.type !== Token.EOF) { + this.getInputStream().consume(); + } + const hasListener = this._parseListeners !== null && this._parseListeners.length > 0; + if (this.buildParseTrees || hasListener) { + let node; + if (this._errHandler.inErrorRecoveryMode(this)) { + node = this._ctx.addErrorNode(o); + } else { + node = this._ctx.addTokenNode(o); + } + node.invokingState = this.state; + if (hasListener) { + this._parseListeners.forEach(function (listener) { + if (node instanceof ErrorNode || (node.isErrorNode !== undefined && node.isErrorNode())) { + listener.visitErrorNode(node); + } else if (node instanceof TerminalNode) { + listener.visitTerminal(node); + } + }); + } + } + return o; + } + + addContextToParseTree() { + // add current context to parent if we have a parent + if (this._ctx.parentCtx !== null) { + this._ctx.parentCtx.addChild(this._ctx); + } + } + + /** + * Always called by generated parsers upon entry to a rule. Access field + * {@link //_ctx} get the current context. + */ + enterRule(localctx, state, ruleIndex) { + this.state = state; + this._ctx = localctx; + this._ctx.start = this._input.LT(1); + if (this.buildParseTrees) { + this.addContextToParseTree(); + } + this.triggerEnterRuleEvent(); + } + + exitRule() { + this._ctx.stop = this._input.LT(-1); + // trigger event on _ctx, before it reverts to parent + this.triggerExitRuleEvent(); + this.state = this._ctx.invokingState; + this._ctx = this._ctx.parentCtx; + } + + enterOuterAlt(localctx, altNum) { + localctx.setAltNumber(altNum); + // if we have new localctx, make sure we replace existing ctx + // that is previous child of parse tree + if (this.buildParseTrees && this._ctx !== localctx) { + if (this._ctx.parentCtx !== null) { + this._ctx.parentCtx.removeLastChild(); + this._ctx.parentCtx.addChild(localctx); + } + } + this._ctx = localctx; + } + + /** + * Get the precedence level for the top-most precedence rule. + * + * @return The precedence level for the top-most precedence rule, or -1 if + * the parser context is not nested within a precedence rule. + */ + getPrecedence() { + if (this._precedenceStack.length === 0) { + return -1; + } else { + return this._precedenceStack[this._precedenceStack.length - 1]; + } + } + + enterRecursionRule(localctx, state, ruleIndex, precedence) { + this.state = state; + this._precedenceStack.push(precedence); + this._ctx = localctx; + this._ctx.start = this._input.LT(1); + this.triggerEnterRuleEvent(); // simulates rule entry for left-recursive rules + } + + // Like {@link //enterRule} but for recursive rules. + pushNewRecursionContext(localctx, state, ruleIndex) { + const previous = this._ctx; + previous.parentCtx = localctx; + previous.invokingState = state; + previous.stop = this._input.LT(-1); + + this._ctx = localctx; + this._ctx.start = previous.start; + if (this.buildParseTrees) { + this._ctx.addChild(previous); + } + this.triggerEnterRuleEvent(); // simulates rule entry for left-recursive rules + } + + unrollRecursionContexts(parentCtx) { + this._precedenceStack.pop(); + this._ctx.stop = this._input.LT(-1); + const retCtx = this._ctx; // save current ctx (return value) + // unroll so _ctx is as it was before call to recursive method + const parseListeners = this.getParseListeners(); + if (parseListeners !== null && parseListeners.length > 0) { + while (this._ctx !== parentCtx) { + this.triggerExitRuleEvent(); + this._ctx = this._ctx.parentCtx; + } + } else { + this._ctx = parentCtx; + } + // hook into tree + retCtx.parentCtx = parentCtx; + if (this.buildParseTrees && parentCtx !== null) { + // add return ctx into invoking rule's tree + parentCtx.addChild(retCtx); + } + } + + getInvokingContext(ruleIndex) { + let ctx = this._ctx; + while (ctx !== null) { + if (ctx.ruleIndex === ruleIndex) { + return ctx; + } + ctx = ctx.parentCtx; + } + return null; + } + + precpred(localctx, precedence) { + return precedence >= this._precedenceStack[this._precedenceStack.length - 1]; + } + + inContext(context) { + // TODO: useful in parser? + return false; + } + + /** + * Checks whether or not {@code symbol} can follow the current state in the + * ATN. The behavior of this method is equivalent to the following, but is + * implemented such that the complete context-sensitive follow set does not + * need to be explicitly constructed. + * + *
+ * return getExpectedTokens().contains(symbol); + *+ * + * @param symbol the symbol type to check + * @return {@code true} if {@code symbol} can follow the current state in + * the ATN, otherwise {@code false}. + */ + isExpectedToken(symbol) { + const atn = this._interp.atn; + let ctx = this._ctx; + const s = atn.states[this.state]; + let following = atn.nextTokens(s); + if (following.contains(symbol)) { + return true; + } + if (!following.contains(Token.EPSILON)) { + return false; + } + while (ctx !== null && ctx.invokingState >= 0 && following.contains(Token.EPSILON)) { + const invokingState = atn.states[ctx.invokingState]; + const rt = invokingState.transitions[0]; + following = atn.nextTokens(rt.followState); + if (following.contains(symbol)) { + return true; + } + ctx = ctx.parentCtx; + } + if (following.contains(Token.EPSILON) && symbol === Token.EOF) { + return true; + } else { + return false; + } + } + + /** + * Computes the set of input symbols which could follow the current parser + * state and context, as given by {@link //getState} and {@link //getContext}, + * respectively. + * + * @see ATN//getExpectedTokens(int, RuleContext) + */ + getExpectedTokens() { + return this._interp.atn.getExpectedTokens(this.state, this._ctx); + } + + getExpectedTokensWithinCurrentRule() { + const atn = this._interp.atn; + const s = atn.states[this.state]; + return atn.nextTokens(s); + } + + // Get a rule's index (i.e., {@code RULE_ruleName} field) or -1 if not found. + getRuleIndex(ruleName) { + const ruleIndex = this.getRuleIndexMap()[ruleName]; + if (ruleIndex !== null) { + return ruleIndex; + } else { + return -1; + } + } + + /** + * Return List<String> of the rule names in your parser instance + * leading up to a call to the current rule. You could override if + * you want more details such as the file/line info of where + * in the ATN a rule is invoked. + * + * this is very useful for error messages. + */ + getRuleInvocationStack(p) { + p = p || null; + if (p === null) { + p = this._ctx; + } + const stack = []; + while (p !== null) { + // compute what follows who invoked us + const ruleIndex = p.ruleIndex; + if (ruleIndex < 0) { + stack.push("n/a"); + } else { + stack.push(this.ruleNames[ruleIndex]); + } + p = p.parentCtx; + } + return stack; + } + + // For debugging and other purposes. + getDFAStrings() { + return this._interp.decisionToDFA.toString(); + } + + // For debugging and other purposes. + dumpDFA() { + let seenOne = false; + for (let i = 0; i < this._interp.decisionToDFA.length; i++) { + const dfa = this._interp.decisionToDFA[i]; + if (dfa.states.length > 0) { + if (seenOne) { + console.log(); + } + this.printer.println("Decision " + dfa.decision + ":"); + this.printer.print(dfa.toString(this.literalNames, this.symbolicNames)); + seenOne = true; + } + } + } + + /* + " printer = function() {\r\n" + + " this.println = function(s) { document.getElementById('output') += s + '\\n'; }\r\n" + + " this.print = function(s) { document.getElementById('output') += s; }\r\n" + + " };\r\n" + + */ + getSourceName() { + return this._input.sourceName; + } + + /** + * During a parse is sometimes useful to listen in on the rule entry and exit + * events as well as token matches. this is for quick and dirty debugging. + */ + setTrace(trace) { + if (!trace) { + this.removeParseListener(this._tracer); + this._tracer = null; + } else { + if (this._tracer !== null) { + this.removeParseListener(this._tracer); + } + this._tracer = new TraceListener(this); + this.addParseListener(this._tracer); + } + } +} + +/** + * this field maps from the serialized ATN string to the deserialized {@link + * ATN} with + * bypass alternatives. + * + * @see ATNDeserializationOptions//isGenerateRuleBypassTransitions() + */ +Parser.bypassAltsAtnCache = {}; diff --git a/packages/antlr4/src/antlr4/Recognizer.d.ts b/packages/antlr4/src/antlr4/Recognizer.d.ts new file mode 100644 index 000000000..d77927c9a --- /dev/null +++ b/packages/antlr4/src/antlr4/Recognizer.d.ts @@ -0,0 +1,9 @@ +import {ErrorListener} from "./error"; + +export declare class Recognizer
This action is implemented by calling {@link Lexer//setChannel} with the + * value provided by {@link //getChannel}.
+ */ + execute(lexer) { + lexer._channel = this.channel; + } + + updateHashCode(hash) { + hash.update(this.actionType, this.channel); + } + + equals(other) { + if (this === other) { + return true; + } else if (! (other instanceof LexerChannelAction)) { + return false; + } else { + return this.channel === other.channel; + } + } + + toString() { + return "channel(" + this.channel + ")"; + } +} + diff --git a/packages/antlr4/src/antlr4/action/LexerCustomAction.js b/packages/antlr4/src/antlr4/action/LexerCustomAction.js new file mode 100644 index 000000000..c40ac0d00 --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerCustomAction.js @@ -0,0 +1,56 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Executes a custom lexer action by calling {@link Recognizer//action} with the + * rule and action indexes assigned to the custom action. The implementation of + * a custom action is added to the generated code for the lexer in an override + * of {@link Recognizer//action} when the grammar is compiled. + * + *This class may represent embedded actions created with the {...}
+ * syntax in ANTLR 4, as well as actions created for lexer commands where the
+ * command argument could not be evaluated when the grammar was compiled.
Custom actions are implemented by calling {@link Lexer//action} with the + * appropriate rule and action indexes.
+ */ + execute(lexer) { + lexer.action(null, this.ruleIndex, this.actionIndex); + } + + updateHashCode(hash) { + hash.update(this.actionType, this.ruleIndex, this.actionIndex); + } + + equals(other) { + if (this === other) { + return true; + } else if (! (other instanceof LexerCustomAction)) { + return false; + } else { + return this.ruleIndex === other.ruleIndex && this.actionIndex === other.actionIndex; + } + } +} diff --git a/packages/antlr4/src/antlr4/action/LexerIndexedCustomAction.js b/packages/antlr4/src/antlr4/action/LexerIndexedCustomAction.js new file mode 100644 index 000000000..6557a5db8 --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerIndexedCustomAction.js @@ -0,0 +1,60 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +/** + * This implementation of {@link LexerAction} is used for tracking input offsets + * for position-dependent actions within a {@link LexerActionExecutor}. + * + *This action is not serialized as part of the ATN, and is only required for + * position-dependent lexer actions which appear at a location other than the + * end of a rule. For more information about DFA optimizations employed for + * lexer actions, see {@link LexerActionExecutor//append} and + * {@link LexerActionExecutor//fixOffsetBeforeMatch}.
+ * + * Constructs a new indexed custom action by associating a character offset + * with a {@link LexerAction}. + * + *Note: This class is only required for lexer actions for which + * {@link LexerAction//isPositionDependent} returns {@code true}.
+ * + * @param offset The offset into the input {@link CharStream}, relative to + * the token start index, at which the specified lexer action should be + * executed. + * @param action The lexer action to execute at a particular offset in the + * input {@link CharStream}. + */ +import LexerAction from "./LexerAction.js"; + + +export default class LexerIndexedCustomAction extends LexerAction { + constructor(offset, action) { + super(action.actionType); + this.offset = offset; + this.action = action; + this.isPositionDependent = true; + } + + /** + *This method calls {@link //execute} on the result of {@link //getAction} + * using the provided {@code lexer}.
+ */ + execute(lexer) { + // assume the input stream position was properly set by the calling code + this.action.execute(lexer); + } + + updateHashCode(hash) { + hash.update(this.actionType, this.offset, this.action); + } + + equals(other) { + if (this === other) { + return true; + } else if (! (other instanceof LexerIndexedCustomAction)) { + return false; + } else { + return this.offset === other.offset && this.action === other.action; + } + } +} diff --git a/packages/antlr4/src/antlr4/action/LexerModeAction.js b/packages/antlr4/src/antlr4/action/LexerModeAction.js new file mode 100644 index 000000000..81f7aca15 --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerModeAction.js @@ -0,0 +1,43 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Implements the {@code mode} lexer action by calling {@link Lexer//mode} with + * the assigned mode + */ +export default class LexerModeAction extends LexerAction { + constructor(mode) { + super(LexerActionType.MODE); + this.mode = mode; + } + + /** + *This action is implemented by calling {@link Lexer//mode} with the + * value provided by {@link //getMode}.
+ */ + execute(lexer) { + lexer.mode(this.mode); + } + + updateHashCode(hash) { + hash.update(this.actionType, this.mode); + } + + equals(other) { + if (this === other) { + return true; + } else if (! (other instanceof LexerModeAction)) { + return false; + } else { + return this.mode === other.mode; + } + } + + toString() { + return "mode(" + this.mode + ")"; + } +} diff --git a/packages/antlr4/src/antlr4/action/LexerMoreAction.js b/packages/antlr4/src/antlr4/action/LexerMoreAction.js new file mode 100644 index 000000000..6dd1885d1 --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerMoreAction.js @@ -0,0 +1,32 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Implements the {@code more} lexer action by calling {@link Lexer//more}. + * + *The {@code more} command does not have any parameters, so this action is + * implemented as a singleton instance exposed by {@link //INSTANCE}.
+ */ +export default class LexerMoreAction extends LexerAction { + constructor() { + super(LexerActionType.MORE); + } + + /** + *This action is implemented by calling {@link Lexer//popMode}.
+ */ + execute(lexer) { + lexer.more(); + } + + toString() { + return "more"; + } +} + +LexerMoreAction.INSTANCE = new LexerMoreAction(); + diff --git a/packages/antlr4/src/antlr4/action/LexerPopModeAction.js b/packages/antlr4/src/antlr4/action/LexerPopModeAction.js new file mode 100644 index 000000000..c6fc7b773 --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerPopModeAction.js @@ -0,0 +1,31 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Implements the {@code popMode} lexer action by calling {@link Lexer//popMode}. + * + *The {@code popMode} command does not have any parameters, so this action is + * implemented as a singleton instance exposed by {@link //INSTANCE}.
+ */ +export default class LexerPopModeAction extends LexerAction { + constructor() { + super(LexerActionType.POP_MODE); + } + + /** + *This action is implemented by calling {@link Lexer//popMode}.
+ */ + execute(lexer) { + lexer.popMode(); + } + + toString() { + return "popMode"; + } +} + +LexerPopModeAction.INSTANCE = new LexerPopModeAction(); diff --git a/packages/antlr4/src/antlr4/action/LexerPushModeAction.js b/packages/antlr4/src/antlr4/action/LexerPushModeAction.js new file mode 100644 index 000000000..2c694b3da --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerPushModeAction.js @@ -0,0 +1,43 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Implements the {@code pushMode} lexer action by calling + * {@link Lexer//pushMode} with the assigned mode + */ +export default class LexerPushModeAction extends LexerAction { + constructor(mode) { + super(LexerActionType.PUSH_MODE); + this.mode = mode; + } + + /** + *This action is implemented by calling {@link Lexer//pushMode} with the + * value provided by {@link //getMode}.
+ */ + execute(lexer) { + lexer.pushMode(this.mode); + } + + updateHashCode(hash) { + hash.update(this.actionType, this.mode); + } + + equals(other) { + if (this === other) { + return true; + } else if (! (other instanceof LexerPushModeAction)) { + return false; + } else { + return this.mode === other.mode; + } + } + + toString() { + return "pushMode(" + this.mode + ")"; + } +} diff --git a/packages/antlr4/src/antlr4/action/LexerSkipAction.js b/packages/antlr4/src/antlr4/action/LexerSkipAction.js new file mode 100644 index 000000000..68a2636fd --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerSkipAction.js @@ -0,0 +1,29 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Implements the {@code skip} lexer action by calling {@link Lexer//skip}. + * + *The {@code skip} command does not have any parameters, so this action is + * implemented as a singleton instance exposed by {@link //INSTANCE}.
+ */ +export default class LexerSkipAction extends LexerAction { + constructor() { + super(LexerActionType.SKIP); + } + + execute(lexer) { + lexer.skip(); + } + + toString() { + return "skip"; + } +} + +// Provides a singleton instance of this parameterless lexer action. +LexerSkipAction.INSTANCE = new LexerSkipAction(); diff --git a/packages/antlr4/src/antlr4/action/LexerTypeAction.js b/packages/antlr4/src/antlr4/action/LexerTypeAction.js new file mode 100644 index 000000000..98092b382 --- /dev/null +++ b/packages/antlr4/src/antlr4/action/LexerTypeAction.js @@ -0,0 +1,41 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import {default as LexerActionType } from "../atn/LexerActionType.js"; +import LexerAction from "./LexerAction.js"; + +/** + * Implements the {@code type} lexer action by calling {@link Lexer//setType} + * with the assigned type + */ + +export default class LexerTypeAction extends LexerAction { + constructor(type) { + super(LexerActionType.TYPE); + this.type = type; + } + + execute(lexer) { + lexer.type = this.type; + } + + updateHashCode(hash) { + hash.update(this.actionType, this.type); + } + + equals(other) { + if(this === other) { + return true; + } else if (! (other instanceof LexerTypeAction)) { + return false; + } else { + return this.type === other.type; + } + } + + toString() { + return "type(" + this.type + ")"; + } +} + diff --git a/packages/antlr4/src/antlr4/atn/ATN.d.ts b/packages/antlr4/src/antlr4/atn/ATN.d.ts new file mode 100644 index 000000000..6140d0a7e --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATN.d.ts @@ -0,0 +1,17 @@ +import {IntervalSet} from "../misc"; +import {RuleContext} from "../context"; +import {ATNState, DecisionState, RuleStartState, RuleStopState} from "../state"; + +export declare class ATN { + + static INVALID_ALT_NUMBER: number; + + states: ATNState[]; + decisionToState: DecisionState[]; + ruleToStartState: RuleStartState[]; + ruleToStopState: RuleStopState[]; + + getExpectedTokens(stateNumber: number, ctx: RuleContext ): IntervalSet; + nextTokens(atnState: ATNState, ctx?: RuleContext): IntervalSet; + +} diff --git a/packages/antlr4/src/antlr4/atn/ATN.js b/packages/antlr4/src/antlr4/atn/ATN.js new file mode 100644 index 000000000..0ddcdca33 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATN.js @@ -0,0 +1,155 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import LL1Analyzer from './LL1Analyzer.js'; +import IntervalSet from '../misc/IntervalSet.js'; +import Token from '../Token.js'; + +export default class ATN { + + constructor(grammarType , maxTokenType) { + /** + * Used for runtime deserialization of ATNs from strings + * The type of the ATN. + */ + this.grammarType = grammarType; + // The maximum value for any symbol recognized by a transition in the ATN. + this.maxTokenType = maxTokenType; + this.states = []; + /** + * Each subrule/rule is a decision point and we must track them so we + * can go back later and build DFA predictors for them. This includes + * all the rules, subrules, optional blocks, ()+, ()* etc... + */ + this.decisionToState = []; + // Maps from rule index to starting state number. + this.ruleToStartState = []; + // Maps from rule index to stop state number. + this.ruleToStopState = null; + this.modeNameToStartState = {}; + /** + * For lexer ATNs, this maps the rule index to the resulting token type. + * For parser ATNs, this maps the rule index to the generated bypass token + * type if the {@link ATNDeserializationOptions//isGenerateRuleBypassTransitions} + * deserialization option was specified; otherwise, this is {@code null} + */ + this.ruleToTokenType = null; + /** + * For lexer ATNs, this is an array of {@link LexerAction} objects which may + * be referenced by action transitions in the ATN + */ + this.lexerActions = null; + this.modeToStartState = []; + } + + /** + * Compute the set of valid tokens that can occur starting in state {@code s}. + * If {@code ctx} is null, the set of tokens will not include what can follow + * the rule surrounding {@code s}. In other words, the set will be + * restricted to tokens reachable staying within {@code s}'s rule + */ + nextTokensInContext(s, ctx) { + const anal = new LL1Analyzer(this); + return anal.LOOK(s, null, ctx); + } + + /** + * Compute the set of valid tokens that can occur starting in {@code s} and + * staying in same rule. {@link Token//EPSILON} is in set if we reach end of + * rule + */ + nextTokensNoContext(s) { + if (s.nextTokenWithinRule !== null ) { + return s.nextTokenWithinRule; + } + s.nextTokenWithinRule = this.nextTokensInContext(s, null); + s.nextTokenWithinRule.readOnly = true; + return s.nextTokenWithinRule; + } + + nextTokens(s, ctx) { + if ( ctx===undefined ) { + return this.nextTokensNoContext(s); + } else { + return this.nextTokensInContext(s, ctx); + } + } + + addState(state) { + if ( state !== null ) { + state.atn = this; + state.stateNumber = this.states.length; + } + this.states.push(state); + } + + removeState(state) { + this.states[state.stateNumber] = null; // just free mem, don't shift states in list + } + + defineDecisionState(s) { + this.decisionToState.push(s); + s.decision = this.decisionToState.length-1; + return s.decision; + } + + getDecisionState(decision) { + if (this.decisionToState.length===0) { + return null; + } else { + return this.decisionToState[decision]; + } + } + + /** + * Computes the set of input symbols which could follow ATN state number + * {@code stateNumber} in the specified full {@code context}. This method + * considers the complete parser context, but does not evaluate semantic + * predicates (i.e. all predicates encountered during the calculation are + * assumed true). If a path in the ATN exists from the starting state to the + * {@link RuleStopState} of the outermost context without matching any + * symbols, {@link Token//EOF} is added to the returned set. + * + *If {@code context} is {@code null}, it is treated as + * {@link ParserRuleContext//EMPTY}.
+ * + * @param stateNumber the ATN state number + * @param ctx the full parse context + * + * @return {IntervalSet} The set of potentially valid input symbols which could follow the + * specified state in the specified context. + * + * @throws IllegalArgumentException if the ATN does not contain a state with + * number {@code stateNumber} + */ + getExpectedTokens(stateNumber, ctx ) { + if ( stateNumber < 0 || stateNumber >= this.states.length ) { + throw("Invalid state number."); + } + const s = this.states[stateNumber]; + let following = this.nextTokens(s); + if (!following.contains(Token.EPSILON)) { + return following; + } + const expected = new IntervalSet(); + expected.addSet(following); + expected.removeOne(Token.EPSILON); + while (ctx !== null && ctx.invokingState >= 0 && following.contains(Token.EPSILON)) { + const invokingState = this.states[ctx.invokingState]; + const rt = invokingState.transitions[0]; + following = this.nextTokens(rt.followState); + expected.addSet(following); + expected.removeOne(Token.EPSILON); + ctx = ctx.parentCtx; + } + if (following.contains(Token.EPSILON)) { + expected.addOne(Token.EOF); + } + return expected; + } +} + +ATN.INVALID_ALT_NUMBER = 0; + diff --git a/packages/antlr4/src/antlr4/atn/ATNConfig.d.ts b/packages/antlr4/src/antlr4/atn/ATNConfig.d.ts new file mode 100644 index 000000000..71029bb87 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNConfig.d.ts @@ -0,0 +1,5 @@ +import { ATNState } from "../state"; + +export declare class ATNConfig { + state: ATNState; +} diff --git a/packages/antlr4/src/antlr4/atn/ATNConfig.js b/packages/antlr4/src/antlr4/atn/ATNConfig.js new file mode 100644 index 000000000..a6ab45b1e --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNConfig.js @@ -0,0 +1,134 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import SemanticContext from './SemanticContext.js'; +import HashCode from "../misc/HashCode.js"; + +function checkParams(params, isCfg) { + if(params===null) { + const result = { state:null, alt:null, context:null, semanticContext:null }; + if(isCfg) { + result.reachesIntoOuterContext = 0; + } + return result; + } else { + const props = {}; + props.state = params.state || null; + props.alt = (params.alt === undefined) ? null : params.alt; + props.context = params.context || null; + props.semanticContext = params.semanticContext || null; + if(isCfg) { + props.reachesIntoOuterContext = params.reachesIntoOuterContext || 0; + props.precedenceFilterSuppressed = params.precedenceFilterSuppressed || false; + } + return props; + } +} + +export default class ATNConfig { + /** + * @param {Object} params A tuple: (ATN state, predicted alt, syntactic, semantic context). + * The syntactic context is a graph-structured stack node whose + * path(s) to the root is the rule invocation(s) + * chain used to arrive at the state. The semantic context is + * the tree of semantic predicates encountered before reaching + * an ATN state + */ + constructor(params, config) { + this.checkContext(params, config); + params = checkParams(params); + config = checkParams(config, true); + // The ATN state associated with this configuration/// + this.state = params.state!==null ? params.state : config.state; + // What alt (or lexer rule) is predicted by this configuration/// + this.alt = params.alt!==null ? params.alt : config.alt; + /** + * The stack of invoking states leading to the rule/states associated + * with this config. We track only those contexts pushed during + * execution of the ATN simulator + */ + this.context = params.context!==null ? params.context : config.context; + this.semanticContext = params.semanticContext!==null ? params.semanticContext : + (config.semanticContext!==null ? config.semanticContext : SemanticContext.NONE); + // TODO: make it a boolean then + /** + * We cannot execute predicates dependent upon local context unless + * we know for sure we are in the correct context. Because there is + * no way to do this efficiently, we simply cannot evaluate + * dependent predicates unless we are in the rule that initially + * invokes the ATN simulator. + * closure() tracks the depth of how far we dip into the + * outer context: depth > 0. Note that it may not be totally + * accurate depth since I don't ever decrement + */ + this.reachesIntoOuterContext = config.reachesIntoOuterContext; + this.precedenceFilterSuppressed = config.precedenceFilterSuppressed; + } + + checkContext(params, config) { + if((params.context===null || params.context===undefined) && + (config===null || config.context===null || config.context===undefined)) { + this.context = null; + } + } + + hashCode() { + const hash = new HashCode(); + this.updateHashCode(hash); + return hash.finish(); + } + + updateHashCode(hash) { + hash.update(this.state.stateNumber, this.alt, this.context, this.semanticContext); + } + + /** + * An ATN configuration is equal to another if both have + * the same state, they predict the same alternative, and + * syntactic/semantic contexts are the same + */ + equals(other) { + if (this === other) { + return true; + } else if (! (other instanceof ATNConfig)) { + return false; + } else { + return this.state.stateNumber===other.state.stateNumber && + this.alt===other.alt && + (this.context===null ? other.context===null : this.context.equals(other.context)) && + this.semanticContext.equals(other.semanticContext) && + this.precedenceFilterSuppressed===other.precedenceFilterSuppressed; + } + } + + hashCodeForConfigSet() { + const hash = new HashCode(); + hash.update(this.state.stateNumber, this.alt, this.semanticContext); + return hash.finish(); + } + + equalsForConfigSet(other) { + if (this === other) { + return true; + } else if (! (other instanceof ATNConfig)) { + return false; + } else { + return this.state.stateNumber===other.state.stateNumber && + this.alt===other.alt && + this.semanticContext.equals(other.semanticContext); + } + } + + toString() { + return "(" + this.state + "," + this.alt + + (this.context!==null ? ",[" + this.context.toString() + "]" : "") + + (this.semanticContext !== SemanticContext.NONE ? + ("," + this.semanticContext.toString()) + : "") + + (this.reachesIntoOuterContext>0 ? + (",up=" + this.reachesIntoOuterContext) + : "") + ")"; + } +} diff --git a/packages/antlr4/src/antlr4/atn/ATNConfigSet.d.ts b/packages/antlr4/src/antlr4/atn/ATNConfigSet.d.ts new file mode 100644 index 000000000..03a8d7c42 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNConfigSet.d.ts @@ -0,0 +1,5 @@ +import {ATNConfig} from "./ATNConfig"; + +export declare class ATNConfigSet { + configs: ATNConfig[]; +} diff --git a/packages/antlr4/src/antlr4/atn/ATNConfigSet.js b/packages/antlr4/src/antlr4/atn/ATNConfigSet.js new file mode 100644 index 000000000..0265d56bc --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNConfigSet.js @@ -0,0 +1,244 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import ATN from './ATN.js'; +import SemanticContext from './SemanticContext.js'; +import { merge } from '../context/PredictionContextUtils.js'; +import arrayToString from "../utils/arrayToString.js"; +import HashSet from "../misc/HashSet.js"; +import equalArrays from "../utils/equalArrays.js"; +import HashCode from "../misc/HashCode.js"; + +function hashATNConfig(c) { + return c.hashCodeForConfigSet(); +} + +function equalATNConfigs(a, b) { + if ( a===b ) { + return true; + } else if ( a===null || b===null ) { + return false; + } else + return a.equalsForConfigSet(b); + } + +/** + * Specialized {@link Set}{@code <}{@link ATNConfig}{@code >} that can track + * info about the set, with support for combining similar configurations using a + * graph-structured stack + */ +export default class ATNConfigSet { + constructor(fullCtx) { + /** + * The reason that we need this is because we don't want the hash map to use + * the standard hash code and equals. We need all configurations with the + * same + * {@code (s,i,_,semctx)} to be equal. Unfortunately, this key effectively + * doubles + * the number of objects associated with ATNConfigs. The other solution is + * to + * use a hash table that lets us specify the equals/hashcode operation. + * All configs but hashed by (s, i, _, pi) not including context. Wiped out + * when we go readonly as this set becomes a DFA state + */ + this.configLookup = new HashSet(hashATNConfig, equalATNConfigs); + /** + * Indicates that this configuration set is part of a full context + * LL prediction. It will be used to determine how to merge $. With SLL + * it's a wildcard whereas it is not for LL context merge + */ + this.fullCtx = fullCtx === undefined ? true : fullCtx; + /** + * Indicates that the set of configurations is read-only. Do not + * allow any code to manipulate the set; DFA states will point at + * the sets and they must not change. This does not protect the other + * fields; in particular, conflictingAlts is set after + * we've made this readonly + */ + this.readOnly = false; + // Track the elements as they are added to the set; supports get(i)/// + this.configs = []; + + // TODO: these fields make me pretty uncomfortable but nice to pack up info + // together, saves recomputation + // TODO: can we track conflicts as they are added to save scanning configs + // later? + this.uniqueAlt = 0; + this.conflictingAlts = null; + + /** + * Used in parser and lexer. In lexer, it indicates we hit a pred + * while computing a closure operation. Don't make a DFA state from this + */ + this.hasSemanticContext = false; + this.dipsIntoOuterContext = false; + + this.cachedHashCode = -1; + } + + /** + * Adding a new config means merging contexts with existing configs for + * {@code (s, i, pi, _)}, where {@code s} is the + * {@link ATNConfig//state}, {@code i} is the {@link ATNConfig//alt}, and + * {@code pi} is the {@link ATNConfig//semanticContext}. We use + * {@code (s,i,pi)} as key. + * + *This method updates {@link //dipsIntoOuterContext} and + * {@link //hasSemanticContext} when necessary.
+ */ + add(config, mergeCache) { + if (mergeCache === undefined) { + mergeCache = null; + } + if (this.readOnly) { + throw "This set is readonly"; + } + if (config.semanticContext !== SemanticContext.NONE) { + this.hasSemanticContext = true; + } + if (config.reachesIntoOuterContext > 0) { + this.dipsIntoOuterContext = true; + } + const existing = this.configLookup.add(config); + if (existing === config) { + this.cachedHashCode = -1; + this.configs.push(config); // track order here + return true; + } + // a previous (s,i,pi,_), merge with it and save result + const rootIsWildcard = !this.fullCtx; + const merged = merge(existing.context, config.context, rootIsWildcard, mergeCache); + /** + * no need to check for existing.context, config.context in cache + * since only way to create new graphs is "call rule" and here. We + * cache at both places + */ + existing.reachesIntoOuterContext = Math.max( existing.reachesIntoOuterContext, config.reachesIntoOuterContext); + // make sure to preserve the precedence filter suppression during the merge + if (config.precedenceFilterSuppressed) { + existing.precedenceFilterSuppressed = true; + } + existing.context = merged; // replace context; no need to alt mapping + return true; + } + + getStates() { + const states = new HashSet(); + for (let i = 0; i < this.configs.length; i++) { + states.add(this.configs[i].state); + } + return states; + } + + getPredicates() { + const preds = []; + for (let i = 0; i < this.configs.length; i++) { + const c = this.configs[i].semanticContext; + if (c !== SemanticContext.NONE) { + preds.push(c.semanticContext); + } + } + return preds; + } + + optimizeConfigs(interpreter) { + if (this.readOnly) { + throw "This set is readonly"; + } + if (this.configLookup.length === 0) { + return; + } + for (let i = 0; i < this.configs.length; i++) { + const config = this.configs[i]; + config.context = interpreter.getCachedContext(config.context); + } + } + + addAll(coll) { + for (let i = 0; i < coll.length; i++) { + this.add(coll[i]); + } + return false; + } + + equals(other) { + return this === other || + (other instanceof ATNConfigSet && + equalArrays(this.configs, other.configs) && + this.fullCtx === other.fullCtx && + this.uniqueAlt === other.uniqueAlt && + this.conflictingAlts === other.conflictingAlts && + this.hasSemanticContext === other.hasSemanticContext && + this.dipsIntoOuterContext === other.dipsIntoOuterContext); + } + + hashCode() { + const hash = new HashCode(); + hash.update(this.configs); + return hash.finish(); + } + + updateHashCode(hash) { + if (this.readOnly) { + if (this.cachedHashCode === -1) { + this.cachedHashCode = this.hashCode(); + } + hash.update(this.cachedHashCode); + } else { + hash.update(this.hashCode()); + } + } + + isEmpty() { + return this.configs.length === 0; + } + + contains(item) { + if (this.configLookup === null) { + throw "This method is not implemented for readonly sets."; + } + return this.configLookup.contains(item); + } + + containsFast(item) { + if (this.configLookup === null) { + throw "This method is not implemented for readonly sets."; + } + return this.configLookup.containsFast(item); + } + + clear() { + if (this.readOnly) { + throw "This set is readonly"; + } + this.configs = []; + this.cachedHashCode = -1; + this.configLookup = new HashSet(); + } + + setReadonly(readOnly) { + this.readOnly = readOnly; + if (readOnly) { + this.configLookup = null; // can't mod, no need for lookup cache + } + } + + toString() { + return arrayToString(this.configs) + + (this.hasSemanticContext ? ",hasSemanticContext=" + this.hasSemanticContext : "") + + (this.uniqueAlt !== ATN.INVALID_ALT_NUMBER ? ",uniqueAlt=" + this.uniqueAlt : "") + + (this.conflictingAlts !== null ? ",conflictingAlts=" + this.conflictingAlts : "") + + (this.dipsIntoOuterContext ? ",dipsIntoOuterContext" : ""); + } + + get items(){ + return this.configs; + } + + get length(){ + return this.configs.length; + } +} + diff --git a/packages/antlr4/src/antlr4/atn/ATNDeserializationOptions.d.ts b/packages/antlr4/src/antlr4/atn/ATNDeserializationOptions.d.ts new file mode 100644 index 000000000..59c159c45 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNDeserializationOptions.d.ts @@ -0,0 +1,6 @@ +export declare class ATNDeserializationOptions { + readOnly?: boolean; + verifyATN?: boolean; + generateRuleBypassTransitions?: boolean; + +} diff --git a/packages/antlr4/src/antlr4/atn/ATNDeserializationOptions.js b/packages/antlr4/src/antlr4/atn/ATNDeserializationOptions.js new file mode 100644 index 000000000..48fe8cf56 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNDeserializationOptions.js @@ -0,0 +1,19 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +export default class ATNDeserializationOptions { + constructor(copyFrom) { + if(copyFrom===undefined) { + copyFrom = null; + } + this.readOnly = false; + this.verifyATN = copyFrom===null ? true : copyFrom.verifyATN; + this.generateRuleBypassTransitions = copyFrom===null ? false : copyFrom.generateRuleBypassTransitions; + } +} + +ATNDeserializationOptions.defaultOptions = new ATNDeserializationOptions(); +ATNDeserializationOptions.defaultOptions.readOnly = true; + diff --git a/packages/antlr4/src/antlr4/atn/ATNDeserializer.d.ts b/packages/antlr4/src/antlr4/atn/ATNDeserializer.d.ts new file mode 100644 index 000000000..92085d4cd --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNDeserializer.d.ts @@ -0,0 +1,7 @@ +import {ATNDeserializationOptions} from "./ATNDeserializationOptions"; +import {ATN} from "./ATN"; + +export declare class ATNDeserializer { + constructor(options?: ATNDeserializationOptions); + deserialize(data: number[]) : ATN; +} diff --git a/packages/antlr4/src/antlr4/atn/ATNDeserializer.js b/packages/antlr4/src/antlr4/atn/ATNDeserializer.js new file mode 100644 index 000000000..66ca12d34 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNDeserializer.js @@ -0,0 +1,603 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import Token from '../Token.js'; +import ATN from './ATN.js'; +import ATNType from './ATNType.js'; + +import ATNState from '../state/ATNState.js'; +import BasicState from '../state/BasicState.js'; +import DecisionState from '../state/DecisionState.js'; +import BlockStartState from '../state/BlockStartState.js'; +import BlockEndState from '../state/BlockEndState.js'; +import LoopEndState from '../state/LoopEndState.js'; +import RuleStartState from '../state/RuleStartState.js'; +import RuleStopState from '../state/RuleStopState.js'; +import TokensStartState from '../state/TokensStartState.js'; +import PlusLoopbackState from '../state/PlusLoopbackState.js'; +import StarLoopbackState from '../state/StarLoopbackState.js'; +import StarLoopEntryState from '../state/StarLoopEntryState.js'; +import PlusBlockStartState from '../state/PlusBlockStartState.js'; +import StarBlockStartState from '../state/StarBlockStartState.js'; +import BasicBlockStartState from '../state/BasicBlockStartState.js'; + +import Transition from '../transition/Transition.js'; +import AtomTransition from '../transition/AtomTransition.js'; +import SetTransition from '../transition/SetTransition.js'; +import NotSetTransition from '../transition/NotSetTransition.js'; +import RuleTransition from '../transition/RuleTransition.js'; +import RangeTransition from '../transition/RangeTransition.js'; +import ActionTransition from '../transition/ActionTransition.js'; +import EpsilonTransition from '../transition/EpsilonTransition.js'; +import WildcardTransition from '../transition/WildcardTransition.js'; +import PredicateTransition from '../transition/PredicateTransition.js'; +import PrecedencePredicateTransition from '../transition/PrecedencePredicateTransition.js'; + + +import IntervalSet from '../misc/IntervalSet.js'; +import ATNDeserializationOptions from './ATNDeserializationOptions.js'; + +import LexerActionType from './LexerActionType.js'; +import LexerSkipAction from '../action/LexerSkipAction.js'; +import LexerChannelAction from '../action/LexerChannelAction.js'; +import LexerCustomAction from '../action/LexerCustomAction.js'; +import LexerMoreAction from '../action/LexerMoreAction.js'; +import LexerTypeAction from '../action/LexerTypeAction.js'; +import LexerPushModeAction from '../action/LexerPushModeAction.js'; +import LexerPopModeAction from '../action/LexerPopModeAction.js'; +import LexerModeAction from '../action/LexerModeAction.js'; + +const SERIALIZED_VERSION = 4; + +function initArray( length, value) { + const tmp = []; + tmp[length-1] = value; + return tmp.map(function(i) {return value;}); +} + +export default class ATNDeserializer { + + constructor(options) { + if ( options=== undefined || options === null ) { + options = ATNDeserializationOptions.defaultOptions; + } + this.deserializationOptions = options; + this.stateFactories = null; + this.actionFactories = null; + } + + deserialize(data) { + const legacy = this.reset(data); + this.checkVersion(legacy); + if(legacy) + this.skipUUID(); + const atn = this.readATN(); + this.readStates(atn, legacy); + this.readRules(atn, legacy); + this.readModes(atn); + const sets = []; + this.readSets(atn, sets, this.readInt.bind(this)); + if(legacy) + this.readSets(atn, sets, this.readInt32.bind(this)); + this.readEdges(atn, sets); + this.readDecisions(atn); + this.readLexerActions(atn, legacy); + this.markPrecedenceDecisions(atn); + this.verifyATN(atn); + if (this.deserializationOptions.generateRuleBypassTransitions && atn.grammarType === ATNType.PARSER ) { + this.generateRuleBypassTransitions(atn); + // re-verify after modification + this.verifyATN(atn); + } + return atn; + } + + reset(data) { + const version = data.charCodeAt ? data.charCodeAt(0) : data[0]; + if(version === SERIALIZED_VERSION - 1) { + const adjust = function (c) { + const v = c.charCodeAt(0); + return v > 1 ? v - 2 : v + 65534; + }; + const temp = data.split("").map(adjust); + // don't adjust the first value since that's the version number + temp[0] = data.charCodeAt(0); + this.data = temp; + this.pos = 0; + return true; + } else { + this.data = data + this.pos = 0; + return false; + } + } + + skipUUID() { + let count = 0; + while(count++ < 8) + this.readInt(); + } + + checkVersion(legacy) { + const version = this.readInt(); + if ( !legacy && version !== SERIALIZED_VERSION ) { + throw ("Could not deserialize ATN with version " + version + " (expected " + SERIALIZED_VERSION + ")."); + } + } + + readATN() { + const grammarType = this.readInt(); + const maxTokenType = this.readInt(); + return new ATN(grammarType, maxTokenType); + } + + readStates(atn, legacy) { + let j, pair, stateNumber; + const loopBackStateNumbers = []; + const endStateNumbers = []; + const nstates = this.readInt(); + for(let i=0; iThis cache makes a huge difference in memory and a little bit in speed. + * For the Java grammar on java.*, it dropped the memory requirements + * at the end from 25M to 16M. We don't store any of the full context + * graphs in the DFA because they are limited to local context only, + * but apparently there's a lot of repetition there as well. We optimize + * the config contexts before storing the config set in the DFA states + * by literally rebuilding them with cached subgraphs only.
+ * + *I tried a cache for use during closure operations, that was + * whacked after each adaptivePredict(). It cost a little bit + * more time I think and doesn't save on the overall footprint + * so it's not worth the complexity.
+ */ + this.atn = atn; + this.sharedContextCache = sharedContextCache; + return this; + } + + getCachedContext(context) { + if (this.sharedContextCache ===null) { + return context; + } + const visited = new HashMap(); + return getCachedPredictionContext(context, this.sharedContextCache, visited); + } +} + +// Must distinguish between missing edge and edge we know leads nowhere/// +ATNSimulator.ERROR = new DFAState(0x7FFFFFFF, new ATNConfigSet()); + diff --git a/packages/antlr4/src/antlr4/atn/ATNType.js b/packages/antlr4/src/antlr4/atn/ATNType.js new file mode 100644 index 000000000..9516fda05 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ATNType.js @@ -0,0 +1,13 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +/** + * Represents the type of recognizer an ATN applies to + */ +export default { + LEXER: 0, + PARSER: 1 +}; + diff --git a/packages/antlr4/src/antlr4/atn/AbstractPredicateTransition.js b/packages/antlr4/src/antlr4/atn/AbstractPredicateTransition.js new file mode 100644 index 000000000..0026fded9 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/AbstractPredicateTransition.js @@ -0,0 +1,11 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import Transition from "../transition/Transition.js"; + +export default class AbstractPredicateTransition extends Transition { + constructor(target) { + super(target); + } +} diff --git a/packages/antlr4/src/antlr4/atn/LL1Analyzer.js b/packages/antlr4/src/antlr4/atn/LL1Analyzer.js new file mode 100644 index 000000000..7e6d9ed76 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/LL1Analyzer.js @@ -0,0 +1,193 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import Token from '../Token.js'; +import ATNConfig from './ATNConfig.js'; +import IntervalSet from '../misc/IntervalSet.js'; +import RuleStopState from '../state/RuleStopState.js'; +import RuleTransition from '../transition/RuleTransition.js'; +import NotSetTransition from '../transition/NotSetTransition.js'; +import WildcardTransition from '../transition/WildcardTransition.js'; +import AbstractPredicateTransition from './AbstractPredicateTransition.js'; +import { predictionContextFromRuleContext } from '../context/PredictionContextUtils.js'; +import PredictionContext from '../context/PredictionContext.js'; +import SingletonPredictionContext from '../context/SingletonPredictionContext.js'; +import BitSet from "../misc/BitSet.js"; +import HashSet from "../misc/HashSet.js"; + +export default class LL1Analyzer { + constructor(atn) { + this.atn = atn; + } + + /** + * Calculates the SLL(1) expected lookahead set for each outgoing transition + * of an {@link ATNState}. The returned array has one element for each + * outgoing transition in {@code s}. If the closure from transition + * i leads to a semantic predicate before matching a symbol, the + * element at index i of the result will be {@code null}. + * + * @param s the ATN state + * @return the expected symbols for each outgoing transition of {@code s}. + */ + getDecisionLookahead(s) { + if (s === null) { + return null; + } + const count = s.transitions.length; + const look = []; + for(let alt=0; alt< count; alt++) { + look[alt] = new IntervalSet(); + const lookBusy = new HashSet(); + const seeThruPreds = false; // fail to get lookahead upon pred + this._LOOK(s.transition(alt).target, null, PredictionContext.EMPTY, + look[alt], lookBusy, new BitSet(), seeThruPreds, false); + // Wipe out lookahead for this alternative if we found nothing + // or we had a predicate when we !seeThruPreds + if (look[alt].length===0 || look[alt].contains(LL1Analyzer.HIT_PRED)) { + look[alt] = null; + } + } + return look; + } + + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *If {@code ctx} is {@code null} and the end of the rule containing + * {@code s} is reached, {@link Token//EPSILON} is added to the result set. + * If {@code ctx} is not {@code null} and the end of the outermost rule is + * reached, {@link Token//EOF} is added to the result set.
+ * + * @param s the ATN state + * @param stopState the ATN state to stop at. This can be a + * {@link BlockEndState} to detect epsilon paths through a closure. + * @param ctx the complete parser context, or {@code null} if the context + * should be ignored + * + * @return The set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + */ + LOOK(s, stopState, ctx) { + const r = new IntervalSet(); + const seeThruPreds = true; // ignore preds; get all lookahead + ctx = ctx || null; + const lookContext = ctx!==null ? predictionContextFromRuleContext(s.atn, ctx) : null; + this._LOOK(s, stopState, lookContext, r, new HashSet(), new BitSet(), seeThruPreds, true); + return r; + } + + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *If {@code ctx} is {@code null} and {@code stopState} or the end of the + * rule containing {@code s} is reached, {@link Token//EPSILON} is added to + * the result set. If {@code ctx} is not {@code null} and {@code addEOF} is + * {@code true} and {@code stopState} or the end of the outermost rule is + * reached, {@link Token//EOF} is added to the result set.
+ * + * @param s the ATN state. + * @param stopState the ATN state to stop at. This can be a + * {@link BlockEndState} to detect epsilon paths through a closure. + * @param ctx The outer context, or {@code null} if the outer context should + * not be used. + * @param look The result lookahead set. + * @param lookBusy A set used for preventing epsilon closures in the ATN + * from causing a stack overflow. Outside code should pass + * {@code new CustomizedSetWe track these variables separately for the DFA and ATN simulation + * because the DFA simulation often has to fail over to the ATN + * simulation. If the ATN simulation fails, we need the DFA to fall + * back to its previously accepted state, if any. If the ATN succeeds, + * then the ATN does the accept and the DFA simulator that invoked it + * can simply return the predicted token type.
+ */ + constructor(recog, atn, decisionToDFA, sharedContextCache) { + super(atn, sharedContextCache); + this.decisionToDFA = decisionToDFA; + this.recog = recog; + /** + * The current token's starting index into the character stream. + * Shared across DFA to ATN simulation in case the ATN fails and the + * DFA did not have a previous accept state. In this case, we use the + * ATN-generated exception object + */ + this.startIndex = -1; + // line number 1..n within the input/// + this.line = 1; + /** + * The index of the character relative to the beginning of the line + * 0..n-1 + */ + this.column = 0; + this.mode = Lexer.DEFAULT_MODE; + /** + * Used during DFA/ATN exec to record the most recent accept configuration + * info + */ + this.prevAccept = new SimState(); + } + + copyState(simulator) { + this.column = simulator.column; + this.line = simulator.line; + this.mode = simulator.mode; + this.startIndex = simulator.startIndex; + } + + match(input, mode) { + this.mode = mode; + const mark = input.mark(); + try { + this.startIndex = input.index; + this.prevAccept.reset(); + const dfa = this.decisionToDFA[mode]; + if (dfa.s0 === null) { + return this.matchATN(input); + } else { + return this.execATN(input, dfa.s0); + } + } finally { + input.release(mark); + } + } + + reset() { + this.prevAccept.reset(); + this.startIndex = -1; + this.line = 1; + this.column = 0; + this.mode = Lexer.DEFAULT_MODE; + } + + matchATN(input) { + const startState = this.atn.modeToStartState[this.mode]; + + if (LexerATNSimulator.debug) { + console.log("matchATN mode " + this.mode + " start: " + startState); + } + const old_mode = this.mode; + const s0_closure = this.computeStartState(input, startState); + const suppressEdge = s0_closure.hasSemanticContext; + s0_closure.hasSemanticContext = false; + + const next = this.addDFAState(s0_closure); + if (!suppressEdge) { + this.decisionToDFA[this.mode].s0 = next; + } + + const predict = this.execATN(input, next); + + if (LexerATNSimulator.debug) { + console.log("DFA after matchATN: " + this.decisionToDFA[old_mode].toLexerString()); + } + return predict; + } + + execATN(input, ds0) { + if (LexerATNSimulator.debug) { + console.log("start state closure=" + ds0.configs); + } + if (ds0.isAcceptState) { + // allow zero-length tokens + this.captureSimState(this.prevAccept, input, ds0); + } + let t = input.LA(1); + let s = ds0; // s is current/from DFA state + + for (; ;) { // while more work + if (LexerATNSimulator.debug) { + console.log("execATN loop starting closure: " + s.configs); + } + + /** + * As we move src->trg, src->trg, we keep track of the previous trg to + * avoid looking up the DFA state again, which is expensive. + * If the previous target was already part of the DFA, we might + * be able to avoid doing a reach operation upon t. If s!=null, + * it means that semantic predicates didn't prevent us from + * creating a DFA state. Once we know s!=null, we check to see if + * the DFA state has an edge already for t. If so, we can just reuse + * it's configuration set; there's no point in re-computing it. + * This is kind of like doing DFA simulation within the ATN + * simulation because DFA simulation is really just a way to avoid + * computing reach/closure sets. Technically, once we know that + * we have a previously added DFA state, we could jump over to + * the DFA simulator. But, that would mean popping back and forth + * a lot and making things more complicated algorithmically. + * This optimization makes a lot of sense for loops within DFA. + * A character will take us back to an existing DFA state + * that already has lots of edges out of it. e.g., .* in comments. + * print("Target for:" + str(s) + " and:" + str(t)) + */ + let target = this.getExistingTargetState(s, t); + // print("Existing:" + str(target)) + if (target === null) { + target = this.computeTargetState(input, s, t); + // print("Computed:" + str(target)) + } + if (target === ATNSimulator.ERROR) { + break; + } + // If this is a consumable input element, make sure to consume before + // capturing the accept state so the input index, line, and char + // position accurately reflect the state of the interpreter at the + // end of the token. + if (t !== Token.EOF) { + this.consume(input); + } + if (target.isAcceptState) { + this.captureSimState(this.prevAccept, input, target); + if (t === Token.EOF) { + break; + } + } + t = input.LA(1); + s = target; // flip; current DFA target becomes new src/from state + } + return this.failOrAccept(this.prevAccept, input, s.configs, t); + } + + /** + * Get an existing target state for an edge in the DFA. If the target state + * for the edge has not yet been computed or is otherwise not available, + * this method returns {@code null}. + * + * @param s The current DFA state + * @param t The next input symbol + * @return The existing target DFA state for the given input symbol + * {@code t}, or {@code null} if the target state for this edge is not + * already cached + */ + getExistingTargetState(s, t) { + if (s.edges === null || t < LexerATNSimulator.MIN_DFA_EDGE || t > LexerATNSimulator.MAX_DFA_EDGE) { + return null; + } + + let target = s.edges[t - LexerATNSimulator.MIN_DFA_EDGE]; + if (target === undefined) { + target = null; + } + if (LexerATNSimulator.debug && target !== null) { + console.log("reuse state " + s.stateNumber + " edge to " + target.stateNumber); + } + return target; + } + + /** + * Compute a target state for an edge in the DFA, and attempt to add the + * computed state and corresponding edge to the DFA. + * + * @param input The input stream + * @param s The current DFA state + * @param t The next input symbol + * + * @return The computed target DFA state for the given input symbol + * {@code t}. If {@code t} does not lead to a valid DFA state, this method + * returns {@link //ERROR}. + */ + computeTargetState(input, s, t) { + const reach = new OrderedATNConfigSet(); + // if we don't find an existing DFA state + // Fill reach starting from closure, following t transitions + this.getReachableConfigSet(input, s.configs, reach, t); + + if (reach.items.length === 0) { // we got nowhere on t from s + if (!reach.hasSemanticContext) { + // we got nowhere on t, don't throw out this knowledge; it'd + // cause a failover from DFA later. + this.addDFAEdge(s, t, ATNSimulator.ERROR); + } + // stop when we can't match any more char + return ATNSimulator.ERROR; + } + // Add an edge from s to target DFA found/created for reach + return this.addDFAEdge(s, t, null, reach); + } + + failOrAccept(prevAccept, input, reach, t) { + if (this.prevAccept.dfaState !== null) { + const lexerActionExecutor = prevAccept.dfaState.lexerActionExecutor; + this.accept(input, lexerActionExecutor, this.startIndex, + prevAccept.index, prevAccept.line, prevAccept.column); + return prevAccept.dfaState.prediction; + } else { + // if no accept and EOF is first char, return EOF + if (t === Token.EOF && input.index === this.startIndex) { + return Token.EOF; + } + throw new LexerNoViableAltException(this.recog, input, this.startIndex, reach); + } + } + + /** + * Given a starting configuration set, figure out all ATN configurations + * we can reach upon input {@code t}. Parameter {@code reach} is a return + * parameter. + */ + getReachableConfigSet(input, closure, reach, t) { + // this is used to skip processing for configs which have a lower priority + // than a config that already reached an accept state for the same rule + let skipAlt = ATN.INVALID_ALT_NUMBER; + for (let i = 0; i < closure.items.length; i++) { + const cfg = closure.items[i]; + const currentAltReachedAcceptState = (cfg.alt === skipAlt); + if (currentAltReachedAcceptState && cfg.passedThroughNonGreedyDecision) { + continue; + } + if (LexerATNSimulator.debug) { + console.log("testing %s at %s\n", this.getTokenName(t), cfg + .toString(this.recog, true)); + } + for (let j = 0; j < cfg.state.transitions.length; j++) { + const trans = cfg.state.transitions[j]; // for each transition + const target = this.getReachableTarget(trans, t); + if (target !== null) { + let lexerActionExecutor = cfg.lexerActionExecutor; + if (lexerActionExecutor !== null) { + lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index - this.startIndex); + } + const treatEofAsEpsilon = (t === Token.EOF); + const config = new LexerATNConfig({state: target, lexerActionExecutor: lexerActionExecutor}, cfg); + if (this.closure(input, config, reach, + currentAltReachedAcceptState, true, treatEofAsEpsilon)) { + // any remaining configs for this alt have a lower priority + // than the one that just reached an accept state. + skipAlt = cfg.alt; + } + } + } + } + } + + accept(input, lexerActionExecutor, startIndex, index, line, charPos) { + if (LexerATNSimulator.debug) { + console.log("ACTION %s\n", lexerActionExecutor); + } + // seek to after last char in token + input.seek(index); + this.line = line; + this.column = charPos; + if (lexerActionExecutor !== null && this.recog !== null) { + lexerActionExecutor.execute(this.recog, input, startIndex); + } + } + + getReachableTarget(trans, t) { + if (trans.matches(t, 0, Lexer.MAX_CHAR_VALUE)) { + return trans.target; + } else { + return null; + } + } + + computeStartState(input, p) { + const initialContext = PredictionContext.EMPTY; + const configs = new OrderedATNConfigSet(); + for (let i = 0; i < p.transitions.length; i++) { + const target = p.transitions[i].target; + const cfg = new LexerATNConfig({state: target, alt: i + 1, context: initialContext}, null); + this.closure(input, cfg, configs, false, false, false); + } + return configs; + } + + /** + * Since the alternatives within any lexer decision are ordered by + * preference, this method stops pursuing the closure as soon as an accept + * state is reached. After the first accept state is reached by depth-first + * search from {@code config}, all other (potentially reachable) states for + * this rule would have a lower priority. + * + * @return {Boolean} {@code true} if an accept state is reached, otherwise + * {@code false}. + */ + closure(input, config, configs, + currentAltReachedAcceptState, speculative, treatEofAsEpsilon) { + let cfg = null; + if (LexerATNSimulator.debug) { + console.log("closure(" + config.toString(this.recog, true) + ")"); + } + if (config.state instanceof RuleStopState) { + if (LexerATNSimulator.debug) { + if (this.recog !== null) { + console.log("closure at %s rule stop %s\n", this.recog.ruleNames[config.state.ruleIndex], config); + } else { + console.log("closure at rule stop %s\n", config); + } + } + if (config.context === null || config.context.hasEmptyPath()) { + if (config.context === null || config.context.isEmpty()) { + configs.add(config); + return true; + } else { + configs.add(new LexerATNConfig({state: config.state, context: PredictionContext.EMPTY}, config)); + currentAltReachedAcceptState = true; + } + } + if (config.context !== null && !config.context.isEmpty()) { + for (let i = 0; i < config.context.length; i++) { + if (config.context.getReturnState(i) !== PredictionContext.EMPTY_RETURN_STATE) { + const newContext = config.context.getParent(i); // "pop" return state + const returnState = this.atn.states[config.context.getReturnState(i)]; + cfg = new LexerATNConfig({state: returnState, context: newContext}, config); + currentAltReachedAcceptState = this.closure(input, cfg, + configs, currentAltReachedAcceptState, speculative, + treatEofAsEpsilon); + } + } + } + return currentAltReachedAcceptState; + } + // optimization + if (!config.state.epsilonOnlyTransitions) { + if (!currentAltReachedAcceptState || !config.passedThroughNonGreedyDecision) { + configs.add(config); + } + } + for (let j = 0; j < config.state.transitions.length; j++) { + const trans = config.state.transitions[j]; + cfg = this.getEpsilonTarget(input, config, trans, configs, speculative, treatEofAsEpsilon); + if (cfg !== null) { + currentAltReachedAcceptState = this.closure(input, cfg, configs, + currentAltReachedAcceptState, speculative, treatEofAsEpsilon); + } + } + return currentAltReachedAcceptState; + } + + // side-effect: can alter configs.hasSemanticContext + getEpsilonTarget(input, config, trans, + configs, speculative, treatEofAsEpsilon) { + let cfg = null; + if (trans.serializationType === Transition.RULE) { + const newContext = SingletonPredictionContext.create(config.context, trans.followState.stateNumber); + cfg = new LexerATNConfig({state: trans.target, context: newContext}, config); + } else if (trans.serializationType === Transition.PRECEDENCE) { + throw "Precedence predicates are not supported in lexers."; + } else if (trans.serializationType === Transition.PREDICATE) { + // Track traversing semantic predicates. If we traverse, + // we cannot add a DFA state for this "reach" computation + // because the DFA would not test the predicate again in the + // future. Rather than creating collections of semantic predicates + // like v3 and testing them on prediction, v4 will test them on the + // fly all the time using the ATN not the DFA. This is slower but + // semantically it's not used that often. One of the key elements to + // this predicate mechanism is not adding DFA states that see + // predicates immediately afterwards in the ATN. For example, + + // a : ID {p1}? | ID {p2}? ; + + // should create the start state for rule 'a' (to save start state + // competition), but should not create target of ID state. The + // collection of ATN states the following ID references includes + // states reached by traversing predicates. Since this is when we + // test them, we cannot cash the DFA state target of ID. + + if (LexerATNSimulator.debug) { + console.log("EVAL rule " + trans.ruleIndex + ":" + trans.predIndex); + } + configs.hasSemanticContext = true; + if (this.evaluatePredicate(input, trans.ruleIndex, trans.predIndex, speculative)) { + cfg = new LexerATNConfig({state: trans.target}, config); + } + } else if (trans.serializationType === Transition.ACTION) { + if (config.context === null || config.context.hasEmptyPath()) { + // execute actions anywhere in the start rule for a token. + // + // TODO: if the entry rule is invoked recursively, some + // actions may be executed during the recursive call. The + // problem can appear when hasEmptyPath() is true but + // isEmpty() is false. In this case, the config needs to be + // split into two contexts - one with just the empty path + // and another with everything but the empty path. + // Unfortunately, the current algorithm does not allow + // getEpsilonTarget to return two configurations, so + // additional modifications are needed before we can support + // the split operation. + const lexerActionExecutor = LexerActionExecutor.append(config.lexerActionExecutor, + this.atn.lexerActions[trans.actionIndex]); + cfg = new LexerATNConfig({state: trans.target, lexerActionExecutor: lexerActionExecutor}, config); + } else { + // ignore actions in referenced rules + cfg = new LexerATNConfig({state: trans.target}, config); + } + } else if (trans.serializationType === Transition.EPSILON) { + cfg = new LexerATNConfig({state: trans.target}, config); + } else if (trans.serializationType === Transition.ATOM || + trans.serializationType === Transition.RANGE || + trans.serializationType === Transition.SET) { + if (treatEofAsEpsilon) { + if (trans.matches(Token.EOF, 0, Lexer.MAX_CHAR_VALUE)) { + cfg = new LexerATNConfig({state: trans.target}, config); + } + } + } + return cfg; + } + + /** + * Evaluate a predicate specified in the lexer. + * + *If {@code speculative} is {@code true}, this method was called before + * {@link //consume} for the matched character. This method should call + * {@link //consume} before evaluating the predicate to ensure position + * sensitive values, including {@link Lexer//getText}, {@link Lexer//getLine}, + * and {@link Lexer//getcolumn}, properly reflect the current + * lexer state. This method should restore {@code input} and the simulator + * to the original state before returning (i.e. undo the actions made by the + * call to {@link //consume}.
+ * + * @param input The input stream. + * @param ruleIndex The rule containing the predicate. + * @param predIndex The index of the predicate within the rule. + * @param speculative {@code true} if the current index in {@code input} is + * one character before the predicate's location. + * + * @return {@code true} if the specified predicate evaluates to + * {@code true}. + */ + evaluatePredicate(input, ruleIndex, + predIndex, speculative) { + // assume true if no recognizer was provided + if (this.recog === null) { + return true; + } + if (!speculative) { + return this.recog.sempred(null, ruleIndex, predIndex); + } + const savedcolumn = this.column; + const savedLine = this.line; + const index = input.index; + const marker = input.mark(); + try { + this.consume(input); + return this.recog.sempred(null, ruleIndex, predIndex); + } finally { + this.column = savedcolumn; + this.line = savedLine; + input.seek(index); + input.release(marker); + } + } + + captureSimState(settings, input, dfaState) { + settings.index = input.index; + settings.line = this.line; + settings.column = this.column; + settings.dfaState = dfaState; + } + + addDFAEdge(from_, tk, to, cfgs) { + if (to === undefined) { + to = null; + } + if (cfgs === undefined) { + cfgs = null; + } + if (to === null && cfgs !== null) { + // leading to this call, ATNConfigSet.hasSemanticContext is used as a + // marker indicating dynamic predicate evaluation makes this edge + // dependent on the specific input sequence, so the static edge in the + // DFA should be omitted. The target DFAState is still created since + // execATN has the ability to resynchronize with the DFA state cache + // following the predicate evaluation step. + // + // TJP notes: next time through the DFA, we see a pred again and eval. + // If that gets us to a previously created (but dangling) DFA + // state, we can continue in pure DFA mode from there. + // / + const suppressEdge = cfgs.hasSemanticContext; + cfgs.hasSemanticContext = false; + + to = this.addDFAState(cfgs); + + if (suppressEdge) { + return to; + } + } + // add the edge + if (tk < LexerATNSimulator.MIN_DFA_EDGE || tk > LexerATNSimulator.MAX_DFA_EDGE) { + // Only track edges within the DFA bounds + return to; + } + if (LexerATNSimulator.debug) { + console.log("EDGE " + from_ + " -> " + to + " upon " + tk); + } + if (from_.edges === null) { + // make room for tokens 1..n and -1 masquerading as index 0 + from_.edges = []; + } + from_.edges[tk - LexerATNSimulator.MIN_DFA_EDGE] = to; // connect + + return to; + } + + /** + * Add a new DFA state if there isn't one with this set of + * configurations already. This method also detects the first + * configuration containing an ATN rule stop state. Later, when + * traversing the DFA, we will know which rule to accept. + */ + addDFAState(configs) { + const proposed = new DFAState(null, configs); + let firstConfigWithRuleStopState = null; + for (let i = 0; i < configs.items.length; i++) { + const cfg = configs.items[i]; + if (cfg.state instanceof RuleStopState) { + firstConfigWithRuleStopState = cfg; + break; + } + } + if (firstConfigWithRuleStopState !== null) { + proposed.isAcceptState = true; + proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor; + proposed.prediction = this.atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex]; + } + const dfa = this.decisionToDFA[this.mode]; + const existing = dfa.states.get(proposed); + if (existing !== null) { + return existing; + } + const newState = proposed; + newState.stateNumber = dfa.states.length; + configs.setReadonly(true); + newState.configs = configs; + dfa.states.add(newState); + return newState; + } + + getDFA(mode) { + return this.decisionToDFA[mode]; + } + +// Get the text matched so far for the current token. + getText(input) { + // index is first lookahead char, don't include. + return input.getText(this.startIndex, input.index - 1); + } + + consume(input) { + const curChar = input.LA(1); + if (curChar === "\n".charCodeAt(0)) { + this.line += 1; + this.column = 0; + } else { + this.column += 1; + } + input.consume(); + } + + getTokenName(tt) { + if (tt === -1) { + return "EOF"; + } else { + return "'" + String.fromCharCode(tt) + "'"; + } + } +} + +LexerATNSimulator.debug = false; +LexerATNSimulator.dfa_debug = false; + +LexerATNSimulator.MIN_DFA_EDGE = 0; +LexerATNSimulator.MAX_DFA_EDGE = 127; // forces unicode to stay in ATN diff --git a/packages/antlr4/src/antlr4/atn/LexerActionExecutor.js b/packages/antlr4/src/antlr4/atn/LexerActionExecutor.js new file mode 100644 index 000000000..02edfb491 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/LexerActionExecutor.js @@ -0,0 +1,170 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import LexerIndexedCustomAction from '../action/LexerIndexedCustomAction.js'; +import HashCode from "../misc/HashCode.js"; + +export default class LexerActionExecutor { + /** + * Represents an executor for a sequence of lexer actions which traversed during + * the matching operation of a lexer rule (token). + * + *The executor tracks position information for position-dependent lexer actions + * efficiently, ensuring that actions appearing only at the end of the rule do + * not cause bloating of the {@link DFA} created for the lexer.
+ */ + constructor(lexerActions) { + this.lexerActions = lexerActions === null ? [] : lexerActions; + /** + * Caches the result of {@link //hashCode} since the hash code is an element + * of the performance-critical {@link LexerATNConfig//hashCode} operation + */ + this.cachedHashCode = HashCode.hashStuff(lexerActions); // "".join([str(la) for la in + // lexerActions])) + return this; + } + + /** + * Creates a {@link LexerActionExecutor} which encodes the current offset + * for position-dependent lexer actions. + * + *Normally, when the executor encounters lexer actions where + * {@link LexerAction//isPositionDependent} returns {@code true}, it calls + * {@link IntStream//seek} on the input {@link CharStream} to set the input + * position to the end of the current token. This behavior provides + * for efficient DFA representation of lexer actions which appear at the end + * of a lexer rule, even when the lexer rule matches a variable number of + * characters.
+ * + *Prior to traversing a match transition in the ATN, the current offset + * from the token start index is assigned to all position-dependent lexer + * actions which have not already been assigned a fixed offset. By storing + * the offsets relative to the token start index, the DFA representation of + * lexer actions which appear in the middle of tokens remains efficient due + * to sharing among tokens of the same length, regardless of their absolute + * position in the input stream.
+ * + *If the current executor already has offsets assigned to all + * position-dependent lexer actions, the method returns {@code this}.
+ * + * @param offset The current offset to assign to all position-dependent + * lexer actions which do not already have offsets assigned. + * + * @return {LexerActionExecutor} A {@link LexerActionExecutor} which stores input stream offsets + * for all position-dependent lexer actions. + */ + fixOffsetBeforeMatch(offset) { + let updatedLexerActions = null; + for (let i = 0; i < this.lexerActions.length; i++) { + if (this.lexerActions[i].isPositionDependent && + !(this.lexerActions[i] instanceof LexerIndexedCustomAction)) { + if (updatedLexerActions === null) { + updatedLexerActions = this.lexerActions.concat([]); + } + updatedLexerActions[i] = new LexerIndexedCustomAction(offset, + this.lexerActions[i]); + } + } + if (updatedLexerActions === null) { + return this; + } else { + return new LexerActionExecutor(updatedLexerActions); + } + } + + /** + * Execute the actions encapsulated by this executor within the context of a + * particular {@link Lexer}. + * + *This method calls {@link IntStream//seek} to set the position of the + * {@code input} {@link CharStream} prior to calling + * {@link LexerAction//execute} on a position-dependent action. Before the + * method returns, the input position will be restored to the same position + * it was in when the method was invoked.
+ * + * @param lexer The lexer instance. + * @param input The input stream which is the source for the current token. + * When this method is called, the current {@link IntStream//index} for + * {@code input} should be the start of the following token, i.e. 1 + * character past the end of the current token. + * @param startIndex The token start index. This value may be passed to + * {@link IntStream//seek} to set the {@code input} position to the beginning + * of the token. + */ + execute(lexer, input, startIndex) { + let requiresSeek = false; + const stopIndex = input.index; + try { + for (let i = 0; i < this.lexerActions.length; i++) { + let lexerAction = this.lexerActions[i]; + if (lexerAction instanceof LexerIndexedCustomAction) { + const offset = lexerAction.offset; + input.seek(startIndex + offset); + lexerAction = lexerAction.action; + requiresSeek = (startIndex + offset) !== stopIndex; + } else if (lexerAction.isPositionDependent) { + input.seek(stopIndex); + requiresSeek = false; + } + lexerAction.execute(lexer); + } + } finally { + if (requiresSeek) { + input.seek(stopIndex); + } + } + } + + hashCode() { + return this.cachedHashCode; + } + + updateHashCode(hash) { + hash.update(this.cachedHashCode); + } + + equals(other) { + if (this === other) { + return true; + } else if (!(other instanceof LexerActionExecutor)) { + return false; + } else if (this.cachedHashCode != other.cachedHashCode) { + return false; + } else if (this.lexerActions.length != other.lexerActions.length) { + return false; + } else { + const numActions = this.lexerActions.length + for (let idx = 0; idx < numActions; ++idx) { + if (!this.lexerActions[idx].equals(other.lexerActions[idx])) { + return false; + } + } + return true; + } + } + + /** + * Creates a {@link LexerActionExecutor} which executes the actions for + * the input {@code lexerActionExecutor} followed by a specified + * {@code lexerAction}. + * + * @param lexerActionExecutor The executor for actions already traversed by + * the lexer while matching a token within a particular + * {@link LexerATNConfig}. If this is {@code null}, the method behaves as + * though it were an empty executor. + * @param lexerAction The lexer action to execute after the actions + * specified in {@code lexerActionExecutor}. + * + * @return {LexerActionExecutor} A {@link LexerActionExecutor} for executing the combine actions + * of {@code lexerActionExecutor} and {@code lexerAction}. + */ + static append(lexerActionExecutor, lexerAction) { + if (lexerActionExecutor === null) { + return new LexerActionExecutor([ lexerAction ]); + } + const lexerActions = lexerActionExecutor.lexerActions.concat([ lexerAction ]); + return new LexerActionExecutor(lexerActions); + } +} diff --git a/packages/antlr4/src/antlr4/atn/LexerActionType.js b/packages/antlr4/src/antlr4/atn/LexerActionType.js new file mode 100644 index 000000000..1d31b3a7b --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/LexerActionType.js @@ -0,0 +1,22 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +export default { + // The type of a {@link LexerChannelAction} action. + CHANNEL: 0, + // The type of a {@link LexerCustomAction} action + CUSTOM: 1, + // The type of a {@link LexerModeAction} action. + MODE: 2, + //The type of a {@link LexerMoreAction} action. + MORE: 3, + //The type of a {@link LexerPopModeAction} action. + POP_MODE: 4, + //The type of a {@link LexerPushModeAction} action. + PUSH_MODE: 5, + //The type of a {@link LexerSkipAction} action. + SKIP: 6, + //The type of a {@link LexerTypeAction} action. + TYPE: 7 +} diff --git a/packages/antlr4/src/antlr4/atn/OrderedATNConfigSet.js b/packages/antlr4/src/antlr4/atn/OrderedATNConfigSet.js new file mode 100644 index 000000000..ca5854dfe --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/OrderedATNConfigSet.js @@ -0,0 +1,13 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import ATNConfigSet from "./ATNConfigSet.js"; +import HashSet from "../misc/HashSet.js"; + +export default class OrderedATNConfigSet extends ATNConfigSet { + constructor() { + super(); + this.configLookup = new HashSet(); + } +} diff --git a/packages/antlr4/src/antlr4/atn/ParserATNSimulator.d.ts b/packages/antlr4/src/antlr4/atn/ParserATNSimulator.d.ts new file mode 100644 index 000000000..78a1feb84 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/ParserATNSimulator.d.ts @@ -0,0 +1,21 @@ +import {ATNSimulator} from "./ATNSimulator"; +import {ParserRuleContext} from "../context"; +import {TokenStream} from "../TokenStream"; +import {Recognizer} from "../Recognizer"; +import {ATN} from "./ATN"; +import {PredictionContextCache} from "./PredictionContextCache"; +import {DFA} from "../dfa"; +import {PredictionMode} from "./PredictionMode"; +import {Token} from "../Token"; + +export declare class ParserATNSimulator extends ATNSimulator { + + predictionMode: PredictionMode; + decisionToDFA: DFA[]; + atn: ATN; + debug?: boolean; + trace_atn_sim?: boolean; + + constructor(recog: Recognizer+ * The basic complexity of the adaptive strategy makes it harder to understand. + * We begin with ATN simulation to build paths in a DFA. Subsequent prediction + * requests go through the DFA first. If they reach a state without an edge for + * the current symbol, the algorithm fails over to the ATN simulation to + * complete the DFA path for the current input (until it finds a conflict state + * or uniquely predicting state).
+ * + *+ * All of that is done without using the outer context because we want to create + * a DFA that is not dependent upon the rule invocation stack when we do a + * prediction. One DFA works in all contexts. We avoid using context not + * necessarily because it's slower, although it can be, but because of the DFA + * caching problem. The closure routine only considers the rule invocation stack + * created during prediction beginning in the decision rule. For example, if + * prediction occurs without invoking another rule's ATN, there are no context + * stacks in the configurations. When lack of context leads to a conflict, we + * don't know if it's an ambiguity or a weakness in the strong LL(*) parsing + * strategy (versus full LL(*)).
+ * + *+ * When SLL yields a configuration set with conflict, we rewind the input and + * retry the ATN simulation, this time using full outer context without adding + * to the DFA. Configuration context stacks will be the full invocation stacks + * from the start rule. If we get a conflict using full context, then we can + * definitively say we have a true ambiguity for that input sequence. If we + * don't get a conflict, it implies that the decision is sensitive to the outer + * context. (It is not context-sensitive in the sense of context-sensitive + * grammars.)
+ * + *+ * The next time we reach this DFA state with an SLL conflict, through DFA + * simulation, we will again retry the ATN simulation using full context mode. + * This is slow because we can't save the results and have to "interpret" the + * ATN each time we get that input.
+ * + *+ * CACHING FULL CONTEXT PREDICTIONS
+ * + *+ * We could cache results from full context to predicted alternative easily and + * that saves a lot of time but doesn't work in presence of predicates. The set + * of visible predicates from the ATN start state changes depending on the + * context, because closure can fall off the end of a rule. I tried to cache + * tuples (stack context, semantic context, predicted alt) but it was slower + * than interpreting and much more complicated. Also required a huge amount of + * memory. The goal is not to create the world's fastest parser anyway. I'd like + * to keep this algorithm simple. By launching multiple threads, we can improve + * the speed of parsing across a large number of files.
+ * + *+ * There is no strict ordering between the amount of input used by SLL vs LL, + * which makes it really hard to build a cache for full context. Let's say that + * we have input A B C that leads to an SLL conflict with full context X. That + * implies that using X we might only use A B but we could also use A B C D to + * resolve conflict. Input A B C D could predict alternative 1 in one position + * in the input and A B C E could predict alternative 2 in another position in + * input. The conflicting SLL configurations could still be non-unique in the + * full context prediction, which would lead us to requiring more input than the + * original A B C. To make a prediction cache work, we have to track the exact + * input used during the previous prediction. That amounts to a cache that maps + * X to a specific DFA for that context.
+ * + *+ * Something should be done for left-recursive expression predictions. They are + * likely LL(1) + pred eval. Easier to do the whole SLL unless error and retry + * with full LL thing Sam does.
+ * + *+ * AVOIDING FULL CONTEXT PREDICTION
+ * + *+ * We avoid doing full context retry when the outer context is empty, we did not + * dip into the outer context by falling off the end of the decision state rule, + * or when we force SLL mode.
+ * + *+ * As an example of the not dip into outer context case, consider as super + * constructor calls versus function calls. One grammar might look like + * this:
+ * + *
+ * ctorBody
+ * : '{' superCall? stat* '}'
+ * ;
+ *
+ *
+ * + * Or, you might see something like
+ * + *+ * stat + * : superCall ';' + * | expression ';' + * | ... + * ; + *+ * + *
+ * In both cases I believe that no closure operations will dip into the outer + * context. In the first case ctorBody in the worst case will stop at the '}'. + * In the 2nd case it should stop at the ';'. Both cases should stay within the + * entry rule and not dip into the outer context.
+ * + *+ * PREDICATES
+ * + *+ * Predicates are always evaluated if present in either SLL or LL both. SLL and + * LL simulation deals with predicates differently. SLL collects predicates as + * it performs closure operations like ANTLR v3 did. It delays predicate + * evaluation until it reaches and accept state. This allows us to cache the SLL + * ATN simulation whereas, if we had evaluated predicates on-the-fly during + * closure, the DFA state configuration sets would be different and we couldn't + * build up a suitable DFA.
+ * + *+ * When building a DFA accept state during ATN simulation, we evaluate any + * predicates and return the sole semantically valid alternative. If there is + * more than 1 alternative, we report an ambiguity. If there are 0 alternatives, + * we throw an exception. Alternatives without predicates act like they have + * true predicates. The simple way to think about it is to strip away all + * alternatives with false predicates and choose the minimum alternative that + * remains.
+ * + *+ * When we start in the DFA and reach an accept state that's predicated, we test + * those and return the minimum semantically viable alternative. If no + * alternatives are viable, we throw an exception.
+ * + *+ * During full LL ATN simulation, closure always evaluates predicates and + * on-the-fly. This is crucial to reducing the configuration set size during + * closure. It hits a landmine when parsing with the Java grammar, for example, + * without this on-the-fly evaluation.
+ * + *+ * SHARING DFA
+ * + *+ * All instances of the same parser share the same decision DFAs through a + * static field. Each instance gets its own ATN simulator but they share the + * same {@link //decisionToDFA} field. They also share a + * {@link PredictionContextCache} object that makes sure that all + * {@link PredictionContext} objects are shared among the DFA states. This makes + * a big size difference.
+ * + *+ * THREAD SAFETY
+ * + *+ * The {@link ParserATNSimulator} locks on the {@link //decisionToDFA} field when + * it adds a new DFA object to that array. {@link //addDFAEdge} + * locks on the DFA for the current decision when setting the + * {@link DFAState//edges} field. {@link //addDFAState} locks on + * the DFA for the current decision when looking up a DFA state to see if it + * already exists. We must make sure that all requests to add DFA states that + * are equivalent result in the same shared DFA object. This is because lots of + * threads will be trying to update the DFA at once. The + * {@link //addDFAState} method also locks inside the DFA lock + * but this time on the shared context cache when it rebuilds the + * configurations' {@link PredictionContext} objects using cached + * subgraphs/nodes. No other locking occurs, even during DFA simulation. This is + * safe as long as we can guarantee that all threads referencing + * {@code s.edge[t]} get the same physical target {@link DFAState}, or + * {@code null}. Once into the DFA, the DFA simulation does not reference the + * {@link DFA//states} map. It follows the {@link DFAState//edges} field to new + * targets. The DFA simulator will either find {@link DFAState//edges} to be + * {@code null}, to be non-{@code null} and {@code dfa.edges[t]} null, or + * {@code dfa.edges[t]} to be non-null. The + * {@link //addDFAEdge} method could be racing to set the field + * but in either case the DFA simulator works; if {@code null}, and requests ATN + * simulation. It could also race trying to get {@code dfa.edges[t]}, but either + * way it will work because it's not doing a test and set operation.
+ * + *+ * Starting with SLL then failing to combined SLL/LL (Two-Stage + * Parsing)
+ * + *+ * Sam pointed out that if SLL does not give a syntax error, then there is no + * point in doing full LL, which is slower. We only have to try LL if we get a + * syntax error. For maximum speed, Sam starts the parser set to pure SLL + * mode with the {@link BailErrorStrategy}:
+ * + *
+ * parser.{@link Parser//getInterpreter() getInterpreter()}.{@link //setPredictionMode setPredictionMode}{@code (}{@link PredictionMode//SLL}{@code )};
+ * parser.{@link Parser//setErrorHandler setErrorHandler}(new {@link BailErrorStrategy}());
+ *
+ *
+ * + * If it does not get a syntax error, then we're done. If it does get a syntax + * error, we need to retry with the combined SLL/LL strategy.
+ * + *+ * The reason this works is as follows. If there are no SLL conflicts, then the + * grammar is SLL (at least for that input set). If there is an SLL conflict, + * the full LL analysis must yield a set of viable alternatives which is a + * subset of the alternatives reported by SLL. If the LL set is a singleton, + * then the grammar is LL but not SLL. If the LL set is the same size as the SLL + * set, the decision is SLL. If the LL set has size > 1, then that decision + * is truly ambiguous on the current input. If the LL set is smaller, then the + * SLL conflict resolution might choose an alternative that the full LL would + * rule out as a possibility based upon better context information. If that's + * the case, then the SLL parse will definitely get an error because the full LL + * analysis says it's not viable. If SLL conflict resolution chooses an + * alternative within the LL set, them both SLL and LL would choose the same + * alternative because they both choose the minimum of multiple conflicting + * alternatives.
+ * + *+ * Let's say we have a set of SLL conflicting alternatives {@code {1, 2, 3}} and + * a smaller LL set called s. If s is {@code {2, 3}}, then SLL + * parsing will get an error because SLL will pursue alternative 1. If + * s is {@code {1, 2}} or {@code {1, 3}} then both SLL and LL will + * choose the same alternative because alternative one is the minimum of either + * set. If s is {@code {2}} or {@code {3}} then SLL will get a syntax + * error. If s is {@code {1}} then SLL will succeed.
+ * + *+ * Of course, if the input is invalid, then we will get an error for sure in + * both SLL and LL parsing. Erroneous input will therefore require 2 passes over + * the input.
+ */ +export default class ParserATNSimulator extends ATNSimulator { + constructor(parser, atn, decisionToDFA, sharedContextCache) { + super(atn, sharedContextCache); + this.parser = parser; + this.decisionToDFA = decisionToDFA; + // SLL, LL, or LL + exact ambig detection?// + this.predictionMode = PredictionMode.LL; + // LAME globals to avoid parameters!!!!! I need these down deep in predTransition + this._input = null; + this._startIndex = 0; + this._outerContext = null; + this._dfa = null; + /** + * Each prediction operation uses a cache for merge of prediction contexts. + * Don't keep around as it wastes huge amounts of memory. DoubleKeyMap + * isn't synchronized but we're ok since two threads shouldn't reuse same + * parser/atnsim object because it can only handle one input at a time. + * This maps graphs a and b to merged result c. (a,b)→c. We can avoid + * the merge if we ever see a and b again. Note that (b,a)→c should + * also be examined during cache lookup. + */ + this.mergeCache = null; + this.debug = false; + this.debug_closure = false; + this.debug_add = false; + this.trace_atn_sim = false; + this.dfa_debug = false; + this.retry_debug = false; + } + + reset() {} + + adaptivePredict(input, decision, outerContext) { + if (this.debug || this.trace_atn_sim) { + console.log("adaptivePredict decision " + decision + + " exec LA(1)==" + this.getLookaheadName(input) + + " line " + input.LT(1).line + ":" + + input.LT(1).column); + } + this._input = input; + this._startIndex = input.index; + this._outerContext = outerContext; + + const dfa = this.decisionToDFA[decision]; + this._dfa = dfa; + const m = input.mark(); + const index = input.index; + + // Now we are certain to have a specific decision's DFA + // But, do we still need an initial state? + try { + let s0; + if (dfa.precedenceDfa) { + // the start state for a precedence DFA depends on the current + // parser precedence, and is provided by a DFA method. + s0 = dfa.getPrecedenceStartState(this.parser.getPrecedence()); + } else { + // the start state for a "regular" DFA is just s0 + s0 = dfa.s0; + } + if (s0===null) { + if (outerContext===null) { + outerContext = RuleContext.EMPTY; + } + if (this.debug ) { + console.log("predictATN decision " + dfa.decision + + " exec LA(1)==" + this.getLookaheadName(input) + + ", outerContext=" + outerContext.toString(this.parser.ruleNames)); + } + + const fullCtx = false; + let s0_closure = this.computeStartState(dfa.atnStartState, RuleContext.EMPTY, fullCtx); + + if( dfa.precedenceDfa) { + // If this is a precedence DFA, we use applyPrecedenceFilter + // to convert the computed start state to a precedence start + // state. We then use DFA.setPrecedenceStartState to set the + // appropriate start state for the precedence level rather + // than simply setting DFA.s0. + // + dfa.s0.configs = s0_closure; // not used for prediction but useful to know start configs anyway + s0_closure = this.applyPrecedenceFilter(s0_closure); + s0 = this.addDFAState(dfa, new DFAState(null, s0_closure)); + dfa.setPrecedenceStartState(this.parser.getPrecedence(), s0); + } else { + s0 = this.addDFAState(dfa, new DFAState(null, s0_closure)); + dfa.s0 = s0; + } + } + const alt = this.execATN(dfa, s0, input, index, outerContext); + if (this.debug) { + console.log("DFA after predictATN: " + dfa.toString(this.parser.literalNames, this.parser.symbolicNames)); + } + return alt; + } finally { + this._dfa = null; + this.mergeCache = null; // wack cache after each prediction + input.seek(index); + input.release(m); + } + } + + /** + * Performs ATN simulation to compute a predicted alternative based + * upon the remaining input, but also updates the DFA cache to avoid + * having to traverse the ATN again for the same input sequence. + * + * There are some key conditions we're looking for after computing a new + * set of ATN configs (proposed DFA state): + * if the set is empty, there is no viable alternative for current symbol + * does the state uniquely predict an alternative? + * does the state have a conflict that would prevent us from + * putting it on the work list? + * + * We also have some key operations to do: + * add an edge from previous DFA state to potentially new DFA state, D, + * upon current symbol but only if adding to work list, which means in all + * cases except no viable alternative (and possibly non-greedy decisions?) + * collecting predicates and adding semantic context to DFA accept states + * adding rule context to context-sensitive DFA accept states + * consuming an input symbol + * reporting a conflict + * reporting an ambiguity + * reporting a context sensitivity + * reporting insufficient predicates + * + * cover these cases: + * dead end + * single alt + * single alt + preds + * conflict + * conflict + preds + * + */ + execATN(dfa, s0, input, startIndex, outerContext ) { + if (this.debug || this.trace_atn_sim) { + console.log("execATN decision " + dfa.decision + + ", DFA state " + s0 + + ", LA(1)==" + this.getLookaheadName(input) + + " line " + input.LT(1).line + ":" + input.LT(1).column); + } + let alt; + let previousD = s0; + + if (this.debug) { + console.log("s0 = " + s0); + } + let t = input.LA(1); + for(;;) { // while more work + let D = this.getExistingTargetState(previousD, t); + if(D===null) { + D = this.computeTargetState(dfa, previousD, t); + } + if(D===ATNSimulator.ERROR) { + // if any configs in previous dipped into outer context, that + // means that input up to t actually finished entry rule + // at least for SLL decision. Full LL doesn't dip into outer + // so don't need special case. + // We will get an error no matter what so delay until after + // decision; better error message. Also, no reachable target + // ATN states in SLL implies LL will also get nowhere. + // If conflict in states that dip out, choose min since we + // will get error no matter what. + const e = this.noViableAlt(input, outerContext, previousD.configs, startIndex); + input.seek(startIndex); + alt = this.getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previousD.configs, outerContext); + if(alt!==ATN.INVALID_ALT_NUMBER) { + return alt; + } else { + throw e; + } + } + if(D.requiresFullContext && this.predictionMode !== PredictionMode.SLL) { + // IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) + let conflictingAlts = null; + if (D.predicates!==null) { + if (this.debug) { + console.log("DFA state has preds in DFA sim LL failover"); + } + const conflictIndex = input.index; + if(conflictIndex !== startIndex) { + input.seek(startIndex); + } + conflictingAlts = this.evalSemanticContext(D.predicates, outerContext, true); + if (conflictingAlts.length===1) { + if(this.debug) { + console.log("Full LL avoided"); + } + return conflictingAlts.minValue(); + } + if (conflictIndex !== startIndex) { + // restore the index so reporting the fallback to full + // context occurs with the index at the correct spot + input.seek(conflictIndex); + } + } + if (this.dfa_debug) { + console.log("ctx sensitive state " + outerContext +" in " + D); + } + const fullCtx = true; + const s0_closure = this.computeStartState(dfa.atnStartState, outerContext, fullCtx); + this.reportAttemptingFullContext(dfa, conflictingAlts, D.configs, startIndex, input.index); + alt = this.execATNWithFullContext(dfa, D, s0_closure, input, startIndex, outerContext); + return alt; + } + if (D.isAcceptState) { + if (D.predicates===null) { + return D.prediction; + } + const stopIndex = input.index; + input.seek(startIndex); + const alts = this.evalSemanticContext(D.predicates, outerContext, true); + if (alts.length===0) { + throw this.noViableAlt(input, outerContext, D.configs, startIndex); + } else if (alts.length===1) { + return alts.minValue(); + } else { + // report ambiguity after predicate evaluation to make sure the correct set of ambig alts is reported. + this.reportAmbiguity(dfa, D, startIndex, stopIndex, false, alts, D.configs); + return alts.minValue(); + } + } + previousD = D; + + if (t !== Token.EOF) { + input.consume(); + t = input.LA(1); + } + } + } + + /** + * Get an existing target state for an edge in the DFA. If the target state + * for the edge has not yet been computed or is otherwise not available, + * this method returns {@code null}. + * + * @param previousD The current DFA state + * @param t The next input symbol + * @return The existing target DFA state for the given input symbol + * {@code t}, or {@code null} if the target state for this edge is not + * already cached + */ + getExistingTargetState(previousD, t) { + const edges = previousD.edges; + if (edges===null) { + return null; + } else { + return edges[t + 1] || null; + } + } + + /** + * Compute a target state for an edge in the DFA, and attempt to add the + * computed state and corresponding edge to the DFA. + * + * @param dfa The DFA + * @param previousD The current DFA state + * @param t The next input symbol + * + * @return The computed target DFA state for the given input symbol + * {@code t}. If {@code t} does not lead to a valid DFA state, this method + * returns {@link //ERROR + */ + computeTargetState(dfa, previousD, t) { + const reach = this.computeReachSet(previousD.configs, t, false); + if(reach===null) { + this.addDFAEdge(dfa, previousD, t, ATNSimulator.ERROR); + return ATNSimulator.ERROR; + } + // create new target state; we'll add to DFA after it's complete + let D = new DFAState(null, reach); + + const predictedAlt = this.getUniqueAlt(reach); + + if (this.debug) { + const altSubSets = PredictionMode.getConflictingAltSubsets(reach); + console.log("SLL altSubSets=" + arrayToString(altSubSets) + + /*", previous=" + previousD.configs + */ + ", configs=" + reach + + ", predict=" + predictedAlt + + ", allSubsetsConflict=" + + PredictionMode.allSubsetsConflict(altSubSets) + ", conflictingAlts=" + + this.getConflictingAlts(reach)); + } + if (predictedAlt!==ATN.INVALID_ALT_NUMBER) { + // NO CONFLICT, UNIQUELY PREDICTED ALT + D.isAcceptState = true; + D.configs.uniqueAlt = predictedAlt; + D.prediction = predictedAlt; + } else if (PredictionMode.hasSLLConflictTerminatingPrediction(this.predictionMode, reach)) { + // MORE THAN ONE VIABLE ALTERNATIVE + D.configs.conflictingAlts = this.getConflictingAlts(reach); + D.requiresFullContext = true; + // in SLL-only mode, we will stop at this state and return the minimum alt + D.isAcceptState = true; + D.prediction = D.configs.conflictingAlts.minValue(); + } + if (D.isAcceptState && D.configs.hasSemanticContext) { + this.predicateDFAState(D, this.atn.getDecisionState(dfa.decision)); + if( D.predicates!==null) { + D.prediction = ATN.INVALID_ALT_NUMBER; + } + } + // all adds to dfa are done after we've created full D state + D = this.addDFAEdge(dfa, previousD, t, D); + return D; + } + + predicateDFAState(dfaState, decisionState) { + // We need to test all predicates, even in DFA states that + // uniquely predict alternative. + const nalts = decisionState.transitions.length; + // Update DFA so reach becomes accept state with (predicate,alt) + // pairs if preds found for conflicting alts + const altsToCollectPredsFrom = this.getConflictingAltsOrUniqueAlt(dfaState.configs); + const altToPred = this.getPredsForAmbigAlts(altsToCollectPredsFrom, dfaState.configs, nalts); + if (altToPred!==null) { + dfaState.predicates = this.getPredicatePredictions(altsToCollectPredsFrom, altToPred); + dfaState.prediction = ATN.INVALID_ALT_NUMBER; // make sure we use preds + } else { + // There are preds in configs but they might go away + // when OR'd together like {p}? || NONE == NONE. If neither + // alt has preds, resolve to min alt + dfaState.prediction = altsToCollectPredsFrom.minValue(); + } + } + +// comes back with reach.uniqueAlt set to a valid alt + execATNWithFullContext(dfa, D, // how far we got before failing over + s0, + input, + startIndex, + outerContext) { + if (this.debug || this.trace_atn_sim) { + console.log("execATNWithFullContext "+s0); + } + const fullCtx = true; + let foundExactAmbig = false; + let reach; + let previous = s0; + input.seek(startIndex); + let t = input.LA(1); + let predictedAlt = -1; + for (;;) { // while more work + reach = this.computeReachSet(previous, t, fullCtx); + if (reach===null) { + // if any configs in previous dipped into outer context, that + // means that input up to t actually finished entry rule + // at least for LL decision. Full LL doesn't dip into outer + // so don't need special case. + // We will get an error no matter what so delay until after + // decision; better error message. Also, no reachable target + // ATN states in SLL implies LL will also get nowhere. + // If conflict in states that dip out, choose min since we + // will get error no matter what. + const e = this.noViableAlt(input, outerContext, previous, startIndex); + input.seek(startIndex); + const alt = this.getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previous, outerContext); + if(alt!==ATN.INVALID_ALT_NUMBER) { + return alt; + } else { + throw e; + } + } + const altSubSets = PredictionMode.getConflictingAltSubsets(reach); + if(this.debug) { + console.log("LL altSubSets=" + altSubSets + ", predict=" + + PredictionMode.getUniqueAlt(altSubSets) + ", resolvesToJustOneViableAlt=" + + PredictionMode.resolvesToJustOneViableAlt(altSubSets)); + } + reach.uniqueAlt = this.getUniqueAlt(reach); + // unique prediction? + if(reach.uniqueAlt!==ATN.INVALID_ALT_NUMBER) { + predictedAlt = reach.uniqueAlt; + break; + } else if (this.predictionMode !== PredictionMode.LL_EXACT_AMBIG_DETECTION) { + predictedAlt = PredictionMode.resolvesToJustOneViableAlt(altSubSets); + if(predictedAlt !== ATN.INVALID_ALT_NUMBER) { + break; + } + } else { + // In exact ambiguity mode, we never try to terminate early. + // Just keeps scarfing until we know what the conflict is + if (PredictionMode.allSubsetsConflict(altSubSets) && PredictionMode.allSubsetsEqual(altSubSets)) { + foundExactAmbig = true; + predictedAlt = PredictionMode.getSingleViableAlt(altSubSets); + break; + } + // else there are multiple non-conflicting subsets or + // we're not sure what the ambiguity is yet. + // So, keep going. + } + previous = reach; + if( t !== Token.EOF) { + input.consume(); + t = input.LA(1); + } + } + // If the configuration set uniquely predicts an alternative, + // without conflict, then we know that it's a full LL decision + // not SLL. + if (reach.uniqueAlt !== ATN.INVALID_ALT_NUMBER ) { + this.reportContextSensitivity(dfa, predictedAlt, reach, startIndex, input.index); + return predictedAlt; + } + // We do not check predicates here because we have checked them + // on-the-fly when doing full context prediction. + + // + // In non-exact ambiguity detection mode, we might actually be able to + // detect an exact ambiguity, but I'm not going to spend the cycles + // needed to check. We only emit ambiguity warnings in exact ambiguity + // mode. + // + // For example, we might know that we have conflicting configurations. + // But, that does not mean that there is no way forward without a + // conflict. It's possible to have nonconflicting alt subsets as in: + + // altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}] + + // from + // + // [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]), + // (13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])] + // + // In this case, (17,1,[5 $]) indicates there is some next sequence that + // would resolve this without conflict to alternative 1. Any other viable + // next sequence, however, is associated with a conflict. We stop + // looking for input because no amount of further lookahead will alter + // the fact that we should predict alternative 1. We just can't say for + // sure that there is an ambiguity without looking further. + + this.reportAmbiguity(dfa, D, startIndex, input.index, foundExactAmbig, null, reach); + + return predictedAlt; + } + + computeReachSet(closure, t, fullCtx) { + if (this.debug) { + console.log("in computeReachSet, starting closure: " + closure); + } + if( this.mergeCache===null) { + this.mergeCache = new DoubleDict(); + } + const intermediate = new ATNConfigSet(fullCtx); + + // Configurations already in a rule stop state indicate reaching the end + // of the decision rule (local context) or end of the start rule (full + // context). Once reached, these configurations are never updated by a + // closure operation, so they are handled separately for the performance + // advantage of having a smaller intermediate set when calling closure. + // + // For full-context reach operations, separate handling is required to + // ensure that the alternative matching the longest overall sequence is + // chosen when multiple such configurations can match the input. + + let skippedStopStates = null; + + // First figure out where we can reach on input t + for (let i=0; iWhen {@code lookToEndOfRule} is true, this method uses + * {@link ATN//nextTokens} for each configuration in {@code configs} which is + * not already in a rule stop state to see if a rule stop state is reachable + * from the configuration via epsilon-only transitions.
+ * + * @param configs the configuration set to update + * @param lookToEndOfRule when true, this method checks for rule stop states + * reachable by epsilon-only transitions from each configuration in + * {@code configs}. + * + * @return {@code configs} if all configurations in {@code configs} are in a + * rule stop state, otherwise return a new configuration set containing only + * the configurations from {@code configs} which are in a rule stop state + */ + removeAllConfigsNotInRuleStopState(configs, lookToEndOfRule) { + if (PredictionMode.allConfigsInRuleStopStates(configs)) { + return configs; + } + const result = new ATNConfigSet(configs.fullCtx); + for(let i=0; i+ * The prediction context must be considered by this filter to address + * situations like the following. + *
+ *
+ *
+ * grammar TA;
+ * prog: statement* EOF;
+ * statement: letterA | statement letterA 'b' ;
+ * letterA: 'a';
+ *
+ *
+ * + * If the above grammar, the ATN state immediately before the token + * reference {@code 'a'} in {@code letterA} is reachable from the left edge + * of both the primary and closure blocks of the left-recursive rule + * {@code statement}. The prediction context associated with each of these + * configurations distinguishes between them, and prevents the alternative + * which stepped out to {@code prog} (and then back in to {@code statement} + * from being eliminated by the filter. + *
+ * + * @param configs The configuration set computed by + * {@link //computeStartState} as the start state for the DFA. + * @return The transformed configuration set representing the start state + * for a precedence DFA at a particular precedence level (determined by + * calling {@link Parser//getPrecedence}) + */ + applyPrecedenceFilter(configs) { + let config; + const statesFromAlt1 = []; + const configSet = new ATNConfigSet(configs.fullCtx); + for(let i=0; i+ * In some scenarios, the algorithm described above could predict an + * alternative which will result in a {@link FailedPredicateException} in + * the parser. Specifically, this could occur if the only configuration + * capable of successfully parsing to the end of the decision rule is + * blocked by a semantic predicate. By choosing this alternative within + * {@link //adaptivePredict} instead of throwing a + * {@link NoViableAltException}, the resulting + * {@link FailedPredicateException} in the parser will identify the specific + * predicate which is preventing the parser from successfully parsing the + * decision rule, which helps developers identify and correct logic errors + * in semantic predicates. + *
+ * + * @param configs The ATN configurations which were valid immediately before + * the {@link //ERROR} state was reached + * @param outerContext The is the \gamma_0 initial parser context from the paper + * or the parser stack at the instant before prediction commences. + * + * @return The value to return from {@link //adaptivePredict}, or + * {@link ATN//INVALID_ALT_NUMBER} if a suitable alternative was not + * identified and {@link //adaptivePredict} should report an error instead + */ + getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(configs, outerContext) { + const cfgs = this.splitAccordingToSemanticValidity(configs, outerContext); + const semValidConfigs = cfgs[0]; + const semInvalidConfigs = cfgs[1]; + let alt = this.getAltThatFinishedDecisionEntryRule(semValidConfigs); + if (alt!==ATN.INVALID_ALT_NUMBER) { // semantically/syntactically viable path exists + return alt; + } + // Is there a syntactically valid path with a failed pred? + if (semInvalidConfigs.items.length>0) { + alt = this.getAltThatFinishedDecisionEntryRule(semInvalidConfigs); + if (alt!==ATN.INVALID_ALT_NUMBER) { // syntactically viable path exists + return alt; + } + } + return ATN.INVALID_ALT_NUMBER; + } + + getAltThatFinishedDecisionEntryRule(configs) { + const alts = []; + for(let i=0;iIf {@code D} is {@link //ERROR}, this method returns {@link //ERROR} and + * does not change the DFA.
+ * + * @param dfa The dfa + * @param D The DFA state to add + * @return The state stored in the DFA. This will be either the existing + * state if {@code D} is already in the DFA, or {@code D} itself if the + * state was not already present + */ + addDFAState(dfa, D) { + if (D === ATNSimulator.ERROR) { + return D; + } + const existing = dfa.states.get(D); + if(existing!==null) { + if ( this.trace_atn_sim ) console.log("addDFAState " + D + " exists"); + return existing; + } + D.stateNumber = dfa.states.length; + if (! D.configs.readOnly) { + D.configs.optimizeConfigs(this); + D.configs.setReadonly(true); + } + + if ( this.trace_atn_sim ) console.log("addDFAState new " + D); + + dfa.states.add(D); + if (this.debug) { + console.log("adding new DFA state: " + D); + } + return D; + } + + reportAttemptingFullContext(dfa, conflictingAlts, configs, startIndex, stopIndex) { + if (this.debug || this.retry_debug) { + const interval = new Interval(startIndex, stopIndex + 1); + console.log("reportAttemptingFullContext decision=" + dfa.decision + ":" + configs + + ", input=" + this.parser.getTokenStream().getText(interval)); + } + if (this.parser!==null) { + this.parser.getErrorListenerDispatch().reportAttemptingFullContext(this.parser, dfa, startIndex, stopIndex, conflictingAlts, configs); + } + } + + reportContextSensitivity(dfa, prediction, configs, startIndex, stopIndex) { + if (this.debug || this.retry_debug) { + const interval = new Interval(startIndex, stopIndex + 1); + console.log("reportContextSensitivity decision=" + dfa.decision + ":" + configs + + ", input=" + this.parser.getTokenStream().getText(interval)); + } + if (this.parser!==null) { + this.parser.getErrorListenerDispatch().reportContextSensitivity(this.parser, dfa, startIndex, stopIndex, prediction, configs); + } + } + + // If context sensitive parsing, we know it's ambiguity not conflict// + reportAmbiguity(dfa, D, startIndex, stopIndex, + exact, ambigAlts, configs ) { + if (this.debug || this.retry_debug) { + const interval = new Interval(startIndex, stopIndex + 1); + console.log("reportAmbiguity " + ambigAlts + ":" + configs + + ", input=" + this.parser.getTokenStream().getText(interval)); + } + if (this.parser!==null) { + this.parser.getErrorListenerDispatch().reportAmbiguity(this.parser, dfa, startIndex, stopIndex, exact, ambigAlts, configs); + } + } +} diff --git a/packages/antlr4/src/antlr4/atn/PrecedencePredicate.js b/packages/antlr4/src/antlr4/atn/PrecedencePredicate.js new file mode 100644 index 000000000..44f69daea --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/PrecedencePredicate.js @@ -0,0 +1,51 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import SemanticContext from "./SemanticContext.js"; + +export default class PrecedencePredicate extends SemanticContext { + + constructor(precedence) { + super(); + this.precedence = precedence === undefined ? 0 : precedence; + } + + evaluate(parser, outerContext) { + return parser.precpred(outerContext, this.precedence); + } + + evalPrecedence(parser, outerContext) { + if (parser.precpred(outerContext, this.precedence)) { + return SemanticContext.NONE; + } else { + return null; + } + } + + compareTo(other) { + return this.precedence - other.precedence; + } + + updateHashCode(hash) { + hash.update(this.precedence); + } + + equals(other) { + if (this === other) { + return true; + } else if (!(other instanceof PrecedencePredicate)) { + return false; + } else { + return this.precedence === other.precedence; + } + } + + toString() { + return "{" + this.precedence + ">=prec}?"; + } + +} + +// HORRIBLE workaround circular import, avoiding dynamic import +SemanticContext.PrecedencePredicate = PrecedencePredicate; diff --git a/packages/antlr4/src/antlr4/atn/Predicate.js b/packages/antlr4/src/antlr4/atn/Predicate.js new file mode 100644 index 000000000..59dcd9688 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/Predicate.js @@ -0,0 +1,46 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import SemanticContext from "./SemanticContext.js"; + +export default class Predicate extends SemanticContext { + + constructor(ruleIndex, predIndex, isCtxDependent) { + super(); + this.ruleIndex = ruleIndex === undefined ? -1 : ruleIndex; + this.predIndex = predIndex === undefined ? -1 : predIndex; + this.isCtxDependent = isCtxDependent === undefined ? false : isCtxDependent; // e.g., $i ref in pred + } + + evaluate(parser, outerContext) { + const localctx = this.isCtxDependent ? outerContext : null; + return parser.sempred(localctx, this.ruleIndex, this.predIndex); + } + + updateHashCode(hash) { + hash.update(this.ruleIndex, this.predIndex, this.isCtxDependent); + } + + equals(other) { + if (this === other) { + return true; + } else if (!(other instanceof Predicate)) { + return false; + } else { + return this.ruleIndex === other.ruleIndex && + this.predIndex === other.predIndex && + this.isCtxDependent === other.isCtxDependent; + } + } + + toString() { + return "{" + this.ruleIndex + ":" + this.predIndex + "}?"; + } +} + +/** + * The default {@link SemanticContext}, which is semantically equivalent to + * a predicate of the form {@code {true}?} + */ +SemanticContext.NONE = new Predicate(); diff --git a/packages/antlr4/src/antlr4/atn/PredictionContextCache.d.ts b/packages/antlr4/src/antlr4/atn/PredictionContextCache.d.ts new file mode 100644 index 000000000..6a2092df1 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/PredictionContextCache.d.ts @@ -0,0 +1,3 @@ +export declare class PredictionContextCache { + +} diff --git a/packages/antlr4/src/antlr4/atn/PredictionContextCache.js b/packages/antlr4/src/antlr4/atn/PredictionContextCache.js new file mode 100644 index 000000000..2abbe1f4f --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/PredictionContextCache.js @@ -0,0 +1,43 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import PredictionContext from "../context/PredictionContext.js"; +import HashMap from "../misc/HashMap.js"; + +/** + * Used to cache {@link PredictionContext} objects. Its used for the shared + * context cash associated with contexts in DFA states. This cache + * can be used for both lexers and parsers. + */ +export default class PredictionContextCache { + + constructor() { + this.cache = new HashMap(); + } + + /** + * Add a context to the cache and return it. If the context already exists, + * return that one instead and do not add a new context to the cache. + * Protect shared cache from unsafe thread access. + */ + add(ctx) { + if (ctx === PredictionContext.EMPTY) { + return PredictionContext.EMPTY; + } + const existing = this.cache.get(ctx) || null; + if (existing !== null) { + return existing; + } + this.cache.set(ctx, ctx); + return ctx; + } + + get(ctx) { + return this.cache.get(ctx) || null; + } + + get length(){ + return this.cache.length; + } +} diff --git a/packages/antlr4/src/antlr4/atn/PredictionMode.d.ts b/packages/antlr4/src/antlr4/atn/PredictionMode.d.ts new file mode 100644 index 000000000..a4b2453a4 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/PredictionMode.d.ts @@ -0,0 +1,5 @@ +export declare class PredictionMode { + static SLL: number; + static LL: number; + static LL_EXACT_AMBIG_DETECTION: number; +} diff --git a/packages/antlr4/src/antlr4/atn/PredictionMode.js b/packages/antlr4/src/antlr4/atn/PredictionMode.js new file mode 100644 index 000000000..9a1be3db3 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/PredictionMode.js @@ -0,0 +1,565 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import ATN from './ATN.js'; +import RuleStopState from '../state/RuleStopState.js'; +import ATNConfigSet from './ATNConfigSet.js'; +import ATNConfig from './ATNConfig.js'; +import SemanticContext from './SemanticContext.js'; +import BitSet from "../misc/BitSet.js"; +import AltDict from "../misc/AltDict.js"; +import HashCode from "../misc/HashCode.js"; +import HashMap from "../misc/HashMap.js"; + +/** + * This enumeration defines the prediction modes available in ANTLR 4 along with + * utility methods for analyzing configuration sets for conflicts and/or + * ambiguities. + */ +const PredictionMode = { + /** + * The SLL(*) prediction mode. This prediction mode ignores the current + * parser context when making predictions. This is the fastest prediction + * mode, and provides correct results for many grammars. This prediction + * mode is more powerful than the prediction mode provided by ANTLR 3, but + * may result in syntax errors for grammar and input combinations which are + * not SLL. + * + *+ * When using this prediction mode, the parser will either return a correct + * parse tree (i.e. the same parse tree that would be returned with the + * {@link //LL} prediction mode), or it will report a syntax error. If a + * syntax error is encountered when using the {@link //SLL} prediction mode, + * it may be due to either an actual syntax error in the input or indicate + * that the particular combination of grammar and input requires the more + * powerful {@link //LL} prediction abilities to complete successfully.
+ * + *+ * This prediction mode does not provide any guarantees for prediction + * behavior for syntactically-incorrect inputs.
+ */ + SLL: 0, + + /** + * The LL(*) prediction mode. This prediction mode allows the current parser + * context to be used for resolving SLL conflicts that occur during + * prediction. This is the fastest prediction mode that guarantees correct + * parse results for all combinations of grammars with syntactically correct + * inputs. + * + *+ * When using this prediction mode, the parser will make correct decisions + * for all syntactically-correct grammar and input combinations. However, in + * cases where the grammar is truly ambiguous this prediction mode might not + * report a precise answer for exactly which alternatives are + * ambiguous.
+ * + *+ * This prediction mode does not provide any guarantees for prediction + * behavior for syntactically-incorrect inputs.
+ */ + LL: 1, + + /** + * + * The LL(*) prediction mode with exact ambiguity detection. In addition to + * the correctness guarantees provided by the {@link //LL} prediction mode, + * this prediction mode instructs the prediction algorithm to determine the + * complete and exact set of ambiguous alternatives for every ambiguous + * decision encountered while parsing. + * + *+ * This prediction mode may be used for diagnosing ambiguities during + * grammar development. Due to the performance overhead of calculating sets + * of ambiguous alternatives, this prediction mode should be avoided when + * the exact results are not necessary.
+ * + *+ * This prediction mode does not provide any guarantees for prediction + * behavior for syntactically-incorrect inputs.
+ */ + LL_EXACT_AMBIG_DETECTION: 2, + + /** + * + * Computes the SLL prediction termination condition. + * + *+ * This method computes the SLL prediction termination condition for both of + * the following cases.
+ * + *COMBINED SLL+LL PARSING
+ * + *When LL-fallback is enabled upon SLL conflict, correct predictions are + * ensured regardless of how the termination condition is computed by this + * method. Due to the substantially higher cost of LL prediction, the + * prediction should only fall back to LL when the additional lookahead + * cannot lead to a unique SLL prediction.
+ * + *Assuming combined SLL+LL parsing, an SLL configuration set with only + * conflicting subsets should fall back to full LL, even if the + * configuration sets don't resolve to the same alternative (e.g. + * {@code {1,2}} and {@code {3,4}}. If there is at least one non-conflicting + * configuration, SLL could continue with the hopes that more lookahead will + * resolve via one of those non-conflicting configurations.
+ * + *Here's the prediction termination rule them: SLL (for SLL+LL parsing) + * stops when it sees only conflicting configuration subsets. In contrast, + * full LL keeps going when there is uncertainty.
+ * + *HEURISTIC
+ * + *As a heuristic, we stop prediction when we see any conflicting subset + * unless we see a state that only has one alternative associated with it. + * The single-alt-state thing lets prediction continue upon rules like + * (otherwise, it would admit defeat too soon):
+ * + *{@code [12|1|[], 6|2|[], 12|2|[]]. s : (ID | ID ID?) ';' ;}
+ * + *When the ATN simulation reaches the state before {@code ';'}, it has a + * DFA state that looks like: {@code [12|1|[], 6|2|[], 12|2|[]]}. Naturally + * {@code 12|1|[]} and {@code 12|2|[]} conflict, but we cannot stop + * processing this node because alternative to has another way to continue, + * via {@code [6|2|[]]}.
+ * + *It also let's us continue for this rule:
+ * + *{@code [1|1|[], 1|2|[], 8|3|[]] a : A | A | A B ;}
+ * + *After matching input A, we reach the stop state for rule A, state 1. + * State 8 is the state right before B. Clearly alternatives 1 and 2 + * conflict and no amount of further lookahead will separate the two. + * However, alternative 3 will be able to continue and so we do not stop + * working on this state. In the previous example, we're concerned with + * states associated with the conflicting alternatives. Here alt 3 is not + * associated with the conflicting configs, but since we can continue + * looking for input reasonably, don't declare the state done.
+ * + *PURE SLL PARSING
+ * + *To handle pure SLL parsing, all we have to do is make sure that we + * combine stack contexts for configurations that differ only by semantic + * predicate. From there, we can do the usual SLL termination heuristic.
+ * + *PREDICATES IN SLL+LL PARSING
+ * + *SLL decisions don't evaluate predicates until after they reach DFA stop + * states because they need to create the DFA cache that works in all + * semantic situations. In contrast, full LL evaluates predicates collected + * during start state computation so it can ignore predicates thereafter. + * This means that SLL termination detection can totally ignore semantic + * predicates.
+ * + *Implementation-wise, {@link ATNConfigSet} combines stack contexts but not + * semantic predicate contexts so we might see two configurations like the + * following.
+ * + *{@code (s, 1, x, {}), (s, 1, x', {p})}
+ * + *Before testing these configurations against others, we have to merge + * {@code x} and {@code x'} (without modifying the existing configurations). + * For example, we test {@code (x+x')==x''} when looking for conflicts in + * the following configurations.
+ * + *{@code (s, 1, x, {}), (s, 1, x', {p}), (s, 2, x'', {})}
+ * + *If the configuration set has predicates (as indicated by + * {@link ATNConfigSet//hasSemanticContext}), this algorithm makes a copy of + * the configurations to strip out all of the predicates so that a standard + * {@link ATNConfigSet} will merge everything ignoring predicates.
+ */ + hasSLLConflictTerminatingPrediction: function( mode, configs) { + // Configs in rule stop states indicate reaching the end of the decision + // rule (local context) or end of start rule (full context). If all + // configs meet this condition, then none of the configurations is able + // to match additional input so we terminate prediction. + // + if (PredictionMode.allConfigsInRuleStopStates(configs)) { + return true; + } + // pure SLL mode parsing + if (mode === PredictionMode.SLL) { + // Don't bother with combining configs from different semantic + // contexts if we can fail over to full LL; costs more time + // since we'll often fail over anyway. + if (configs.hasSemanticContext) { + // dup configs, tossing out semantic predicates + const dup = new ATNConfigSet(); + for(let i=0;iThe basic idea is to split the set of configurations {@code C}, into + * conflicting subsets {@code (s, _, ctx, _)} and singleton subsets with + * non-conflicting configurations. Two configurations conflict if they have + * identical {@link ATNConfig//state} and {@link ATNConfig//context} values + * but different {@link ATNConfig//alt} value, e.g. {@code (s, i, ctx, _)} + * and {@code (s, j, ctx, _)} for {@code i!=j}.
+ * + *Reduce these configuration subsets to the set of possible alternatives. + * You can compute the alternative subsets in one pass as follows:
+ * + *{@code A_s,ctx = {i | (s, i, ctx, _)}} for each configuration in + * {@code C} holding {@code s} and {@code ctx} fixed.
+ * + *Or in pseudo-code, for each configuration {@code c} in {@code C}:
+ * + *
+ * map[c] U= c.{@link ATNConfig//alt alt} // map hash/equals uses s and x, not
+ * alt and not pred
+ *
+ *
+ * The values in {@code map} are the set of {@code A_s,ctx} sets.
+ * + *If {@code |A_s,ctx|=1} then there is no conflict associated with + * {@code s} and {@code ctx}.
+ * + *Reduce the subsets to singletons by choosing a minimum of each subset. If + * the union of these alternative subsets is a singleton, then no amount of + * more lookahead will help us. We will always pick that alternative. If, + * however, there is more than one alternative, then we are uncertain which + * alternative to predict and must continue looking for resolution. We may + * or may not discover an ambiguity in the future, even if there are no + * conflicting subsets this round.
+ * + *The biggest sin is to terminate early because it means we've made a + * decision but were uncertain as to the eventual outcome. We haven't used + * enough lookahead. On the other hand, announcing a conflict too late is no + * big deal; you will still have the conflict. It's just inefficient. It + * might even look until the end of file.
+ * + *No special consideration for semantic predicates is required because + * predicates are evaluated on-the-fly for full LL prediction, ensuring that + * no configuration contains a semantic context during the termination + * check.
+ * + *CONFLICTING CONFIGS
+ * + *Two configurations {@code (s, i, x)} and {@code (s, j, x')}, conflict + * when {@code i!=j} but {@code x=x'}. Because we merge all + * {@code (s, i, _)} configurations together, that means that there are at + * most {@code n} configurations associated with state {@code s} for + * {@code n} possible alternatives in the decision. The merged stacks + * complicate the comparison of configuration contexts {@code x} and + * {@code x'}. Sam checks to see if one is a subset of the other by calling + * merge and checking to see if the merged result is either {@code x} or + * {@code x'}. If the {@code x} associated with lowest alternative {@code i} + * is the superset, then {@code i} is the only possible prediction since the + * others resolve to {@code min(i)} as well. However, if {@code x} is + * associated with {@code j>i} then at least one stack configuration for + * {@code j} is not in conflict with alternative {@code i}. The algorithm + * should keep going, looking for more lookahead due to the uncertainty.
+ * + *For simplicity, I'm doing a equality check between {@code x} and + * {@code x'} that lets the algorithm continue to consume lookahead longer + * than necessary. The reason I like the equality is of course the + * simplicity but also because that is the test you need to detect the + * alternatives that are actually in conflict.
+ * + *CONTINUE/STOP RULE
+ * + *Continue if union of resolved alternative sets from non-conflicting and + * conflicting alternative subsets has more than one alternative. We are + * uncertain about which alternative to predict.
+ * + *The complete set of alternatives, {@code [i for (_,i,_)]}, tells us which + * alternatives are still in the running for the amount of input we've + * consumed at this point. The conflicting sets let us to strip away + * configurations that won't lead to more states because we resolve + * conflicts to the configuration with a minimum alternate for the + * conflicting set.
+ * + *CASES
+ * + *EXACT AMBIGUITY DETECTION
+ * + *If all states report the same conflicting set of alternatives, then we + * know we have the exact ambiguity set.
+ * + *|A_i|>1 and
+ * A_i = A_j for all i, j.
In other words, we continue examining lookahead until all {@code A_i} + * have more than one alternative and all {@code A_i} are the same. If + * {@code A={{1,2}, {1,3}}}, then regular LL prediction would terminate + * because the resolved set is {@code {1}}. To determine what the real + * ambiguity is, we have to know whether the ambiguity is between one and + * two or one and three so we keep going. We can only stop prediction when + * we need exact ambiguity detection when the sets look like + * {@code A={{1,2}}} or {@code {{1,2},{1,2}}}, etc...
+ */ + resolvesToJustOneViableAlt: function(altsets) { + return PredictionMode.getSingleViableAlt(altsets); + }, + + /** + * Determines if every alternative subset in {@code altsets} contains more + * than one alternative. + * + * @param altsets a collection of alternative subsets + * @return {@code true} if every {@link BitSet} in {@code altsets} has + * {@link BitSet//cardinality cardinality} > 1, otherwise {@code false} + */ + allSubsetsConflict: function(altsets) { + return ! PredictionMode.hasNonConflictingAltSet(altsets); + }, + /** + * Determines if any single alternative subset in {@code altsets} contains + * exactly one alternative. + * + * @param altsets a collection of alternative subsets + * @return {@code true} if {@code altsets} contains a {@link BitSet} with + * {@link BitSet//cardinality cardinality} 1, otherwise {@code false} + */ + hasNonConflictingAltSet: function(altsets) { + for(let i=0;i
+ * map[c.{@link ATNConfig//state state}] U= c.{@link ATNConfig//alt alt}
+ *
+ */
+ getStateToAltMap: function(configs) {
+ const m = new AltDict();
+ configs.items.map(function(c) {
+ let alts = m.get(c.state);
+ if (alts === null) {
+ alts = new BitSet();
+ m.set(c.state, alts);
+ }
+ alts.add(c.alt);
+ });
+ return m;
+ },
+
+ hasStateAssociatedWithOneAlt: function(configs) {
+ const values = PredictionMode.getStateToAltMap(configs).values();
+ for(let i=0;iFor context dependent predicates, we must pass in a local context so that + * references such as $arg evaluate properly as _localctx.arg. We only + * capture context dependent predicates in the context in which we begin + * prediction, so we passed in the outer context here in case of context + * dependent predicate evaluation.
+ */ + evaluate(parser, outerContext) {} + + /** + * Evaluate the precedence predicates for the context and reduce the result. + * + * @param parser The parser instance. + * @param outerContext The current parser context object. + * @return The simplified semantic context after precedence predicates are + * evaluated, which will be one of the following values. + *+ * The evaluation of predicates by this context is short-circuiting, but + * unordered.
+ */ + evaluate(parser, outerContext) { + for (let i = 0; i < this.opnds.length; i++) { + if (this.opnds[i].evaluate(parser, outerContext)) { + return true; + } + } + return false; + } + + evalPrecedence(parser, outerContext) { + let differs = false; + const operands = []; + for (let i = 0; i < this.opnds.length; i++) { + const context = this.opnds[i]; + const evaluated = context.evalPrecedence(parser, outerContext); + differs |= (evaluated !== context); + if (evaluated === SemanticContext.NONE) { + // The OR context is true if any element is true + return SemanticContext.NONE; + } else if (evaluated !== null) { + // Reduce the result by skipping false elements + operands.push(evaluated); + } + } + if (!differs) { + return this; + } + if (operands.length === 0) { + // all elements were false, so the OR context is false + return null; + } + const result = null; + operands.map(function(o) { + return result === null ? o : SemanticContext.orContext(result, o); + }); + return result; + } + + toString() { + const s = this.opnds.map(o => o.toString()); + return (s.length > 3 ? s.slice(3) : s).join("||"); + } +} + +function filterPrecedencePredicates(set) { + const result = []; + set.values().map( function(context) { + if (context instanceof SemanticContext.PrecedencePredicate) { + result.push(context); + } + }); + return result; +} diff --git a/packages/antlr4/src/antlr4/atn/index.d.ts b/packages/antlr4/src/antlr4/atn/index.d.ts new file mode 100644 index 000000000..d17c1f3f9 --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/index.d.ts @@ -0,0 +1,9 @@ +export * from './ATN'; +export * from './ATNConfig'; +export * from './ATNConfigSet'; +export * from './ATNDeserializer'; +export * from './LexerATNSimulator'; +export * from './ParserATNSimulator'; +export * from './PredictionMode'; +export * from './PredictionContextCache'; + diff --git a/packages/antlr4/src/antlr4/atn/index.js b/packages/antlr4/src/antlr4/atn/index.js new file mode 100644 index 000000000..166b5353f --- /dev/null +++ b/packages/antlr4/src/antlr4/atn/index.js @@ -0,0 +1,13 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import ATN from './ATN.js'; +import ATNDeserializer from './ATNDeserializer.js'; +import LexerATNSimulator from './LexerATNSimulator.js'; +import ParserATNSimulator from './ParserATNSimulator.js'; +import PredictionMode from './PredictionMode.js'; +import PredictionContextCache from './PredictionContextCache.js'; + +export default { ATN, ATNDeserializer, LexerATNSimulator, ParserATNSimulator, PredictionMode, PredictionContextCache } diff --git a/packages/antlr4/src/antlr4/context/ArrayPredictionContext.js b/packages/antlr4/src/antlr4/context/ArrayPredictionContext.js new file mode 100644 index 000000000..fc2394c9f --- /dev/null +++ b/packages/antlr4/src/antlr4/context/ArrayPredictionContext.js @@ -0,0 +1,82 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import PredictionContext from "./PredictionContext.js"; +import equalArrays from "../utils/equalArrays.js"; +import HashCode from "../misc/HashCode.js"; + +export default class ArrayPredictionContext extends PredictionContext { + + constructor(parents, returnStates) { + /** + * Parent can be null only if full ctx mode and we make an array + * from {@link //EMPTY} and non-empty. We merge {@link //EMPTY} by using + * null parent and + * returnState == {@link //EMPTY_RETURN_STATE}. + */ + const h = new HashCode(); + h.update(parents, returnStates); + const hashCode = h.finish(); + super(hashCode); + this.parents = parents; + this.returnStates = returnStates; + return this; + } + + isEmpty() { + // since EMPTY_RETURN_STATE can only appear in the last position, we + // don't need to verify that size==1 + return this.returnStates[0] === PredictionContext.EMPTY_RETURN_STATE; + } + + getParent(index) { + return this.parents[index]; + } + + getReturnState(index) { + return this.returnStates[index]; + } + + equals(other) { + if (this === other) { + return true; + } else if (!(other instanceof ArrayPredictionContext)) { + return false; + } else if (this.hashCode() !== other.hashCode()) { + return false; // can't be same if hash is different + } else { + return equalArrays(this.returnStates, other.returnStates) && + equalArrays(this.parents, other.parents); + } + } + + toString() { + if (this.isEmpty()) { + return "[]"; + } else { + let s = "["; + for (let i = 0; i < this.returnStates.length; i++) { + if (i > 0) { + s = s + ", "; + } + if (this.returnStates[i] === PredictionContext.EMPTY_RETURN_STATE) { + s = s + "$"; + continue; + } + s = s + this.returnStates[i]; + if (this.parents[i] !== null) { + s = s + " " + this.parents[i]; + } else { + s = s + "null"; + } + } + return s + "]"; + } + } + + get length(){ + return this.returnStates.length; + } +} + diff --git a/packages/antlr4/src/antlr4/context/EmptyPredictionContext.js b/packages/antlr4/src/antlr4/context/EmptyPredictionContext.js new file mode 100644 index 000000000..f64a56cb1 --- /dev/null +++ b/packages/antlr4/src/antlr4/context/EmptyPredictionContext.js @@ -0,0 +1,36 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import PredictionContext from "./PredictionContext.js"; +import SingletonPredictionContext from "./SingletonPredictionContext.js"; + +export default class EmptyPredictionContext extends SingletonPredictionContext { + + constructor() { + super(null, PredictionContext.EMPTY_RETURN_STATE); + } + + isEmpty() { + return true; + } + + getParent(index) { + return null; + } + + getReturnState(index) { + return this.returnState; + } + + equals(other) { + return this === other; + } + + toString() { + return "$"; + } +} + + +PredictionContext.EMPTY = new EmptyPredictionContext(); diff --git a/packages/antlr4/src/antlr4/context/InterpreterRuleContext.js b/packages/antlr4/src/antlr4/context/InterpreterRuleContext.js new file mode 100644 index 000000000..6648c5fa2 --- /dev/null +++ b/packages/antlr4/src/antlr4/context/InterpreterRuleContext.js @@ -0,0 +1,12 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import ParserRuleContext from "./ParserRuleContext.js"; + +export default class InterpreterRuleContext extends ParserRuleContext { + constructor(parent, invokingStateNumber, ruleIndex) { + super(parent, invokingStateNumber); + this.ruleIndex = ruleIndex; + } +} diff --git a/packages/antlr4/src/antlr4/context/ParserRuleContext.d.ts b/packages/antlr4/src/antlr4/context/ParserRuleContext.d.ts new file mode 100644 index 000000000..36353be16 --- /dev/null +++ b/packages/antlr4/src/antlr4/context/ParserRuleContext.d.ts @@ -0,0 +1,23 @@ +import {RuleContext} from "./RuleContext"; +import {ParseTree, TerminalNode} from "../tree"; +import {RecognitionException} from "../error"; +import {Token} from "../Token"; +import {Parser} from "../Parser"; + +export declare class ParserRuleContext extends RuleContext { + start: Token; + stop: Token | undefined; + children: ParseTree[] | null; + parentCtx: ParserRuleContext | undefined; + exception?: RecognitionException; + parser?: Parser; + + constructor(parent?: ParserRuleContext, invokingStateNumber?: number); + copyFrom(ctx: ParserRuleContext): void; + getChildCount() : number; + getChild(i: number) : ParseTree; + getToken(ttype: number, i: number): TerminalNode; + getTokens(ttype: number): TerminalNode[]; + getTypedRuleContextDifferent tops, different parents.
+ *
Shared top, same parents.
+ *
Shared top, different parents.
+ *
Shared top, all shared parents.
+ *
Equal tops, merge parents and reduce top to
+ * {@link SingletonPredictionContext}.
+ *
Stack tops equal, parents merge is same; return left graph.
+ *
Same stack top, parents differ; merge parents giving array node, then
+ * remainders of those graphs. A new root node is created to point to the
+ * merged parents.
+ *
Different stack tops pointing to same parent. Make array node for the
+ * root where both element in the root point to the same (original)
+ * parent.
+ *
Different stack tops pointing to different parents. Make array node for
+ * the root where each element points to the corresponding original
+ * parent.
+ *
These local-context merge operations are used when {@code rootIsWildcard} + * is true.
+ * + *{@link //EMPTY} is superset of any graph; return {@link //EMPTY}.
+ *
{@link //EMPTY} and anything is {@code //EMPTY}, so merged parent is
+ * {@code //EMPTY}; return left graph.
+ *
Special case of last merge if local context.
+ *
These full-context merge operations are used when {@code rootIsWildcard} + * is false.
+ * + * + * + *Must keep all contexts; {@link //EMPTY} in array is a special value (and
+ * null parent).
+ *
+ * Since tokens on hidden channels (e.g. whitespace or comments) are not + * added to the parse trees, they will not appear in the output of this + * method. + */ + getText() { + if (this.getChildCount() === 0) { + return ""; + } else { + return this.children.map(function (child) { + return child.getText(); + }).join(""); + } + } + + /** + * For rule associated with this parse tree internal node, return + * the outer alternative number used to match the input. Default + * implementation does not compute nor store this alt num. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * to set it. + */ + getAltNumber() { + // use constant value of ATN.INVALID_ALT_NUMBER to avoid circular dependency + return 0; + } + + /** + * Set the outer alternative number for this context node. Default + * implementation does nothing to avoid backing field overhead for + * trees that don't need it. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + */ + setAltNumber(altNumber) { + } + + getChild(i) { + return null; + } + + getChildCount() { + return 0; + } + + accept(visitor) { + return visitor.visitChildren(this); + } + + /** + * Print out a whole tree, not just a node, in LISP format + * (root child1 .. childN). Print just a node if this is a leaf. + */ + toStringTree(ruleNames, recog) { + return Trees.toStringTree(this, ruleNames, recog); + } + + toString(ruleNames, stop) { + ruleNames = ruleNames || null; + stop = stop || null; + let p = this; + let s = "["; + while (p !== null && p !== stop) { + if (ruleNames === null) { + if (!p.isEmpty()) { + s += p.invokingState; + } + } else { + const ri = p.ruleIndex; + const ruleName = (ri >= 0 && ri < ruleNames.length) ? ruleNames[ri] + : "" + ri; + s += ruleName; + } + if (p.parentCtx !== null && (ruleNames !== null || !p.parentCtx.isEmpty())) { + s += " "; + } + p = p.parentCtx; + } + s += "]"; + return s; + } +} diff --git a/packages/antlr4/src/antlr4/context/SingletonPredictionContext.js b/packages/antlr4/src/antlr4/context/SingletonPredictionContext.js new file mode 100644 index 000000000..99e545c3d --- /dev/null +++ b/packages/antlr4/src/antlr4/context/SingletonPredictionContext.js @@ -0,0 +1,75 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import PredictionContext from './PredictionContext.js'; +import HashCode from "../misc/HashCode.js"; + +export default class SingletonPredictionContext extends PredictionContext { + + constructor(parent, returnState) { + let hashCode = 0; + const hash = new HashCode(); + if(parent !== null) { + hash.update(parent, returnState); + } else { + hash.update(1); + } + hashCode = hash.finish(); + super(hashCode); + this.parentCtx = parent; + this.returnState = returnState; + } + + getParent(index) { + return this.parentCtx; + } + + getReturnState(index) { + return this.returnState; + } + + equals(other) { + if (this === other) { + return true; + } else if (!(other instanceof SingletonPredictionContext)) { + return false; + } else if (this.hashCode() !== other.hashCode()) { + return false; // can't be same if hash is different + } else { + if(this.returnState !== other.returnState) + return false; + else if(this.parentCtx==null) + return other.parentCtx==null + else + return this.parentCtx.equals(other.parentCtx); + } + } + + toString() { + const up = this.parentCtx === null ? "" : this.parentCtx.toString(); + if (up.length === 0) { + if (this.returnState === PredictionContext.EMPTY_RETURN_STATE) { + return "$"; + } else { + return "" + this.returnState; + } + } else { + return "" + this.returnState + " " + up; + } + } + + get length(){ + return 1; + } + + static create(parent, returnState) { + if (returnState === PredictionContext.EMPTY_RETURN_STATE && parent === null) { + // someone can pass in the bits of an array ctx that mean $ + return PredictionContext.EMPTY; + } else { + return new SingletonPredictionContext(parent, returnState); + } + } +} + diff --git a/packages/antlr4/src/antlr4/context/index.d.ts b/packages/antlr4/src/antlr4/context/index.d.ts new file mode 100644 index 000000000..e0e941d81 --- /dev/null +++ b/packages/antlr4/src/antlr4/context/index.d.ts @@ -0,0 +1,3 @@ +export * from './RuleContext'; +export * from './ParserRuleContext'; + diff --git a/packages/antlr4/src/antlr4/context/index.js b/packages/antlr4/src/antlr4/context/index.js new file mode 100644 index 000000000..cdc6ae642 --- /dev/null +++ b/packages/antlr4/src/antlr4/context/index.js @@ -0,0 +1,7 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import PredictionContext from './PredictionContext.js'; + +export default { PredictionContext } diff --git a/packages/antlr4/src/antlr4/dfa/DFA.d.ts b/packages/antlr4/src/antlr4/dfa/DFA.d.ts new file mode 100644 index 000000000..986d7e5a2 --- /dev/null +++ b/packages/antlr4/src/antlr4/dfa/DFA.d.ts @@ -0,0 +1,5 @@ +export declare class DFA { + constructor(ds: any, index: number); + toLexerString(): string; +} + diff --git a/packages/antlr4/src/antlr4/dfa/DFA.js b/packages/antlr4/src/antlr4/dfa/DFA.js new file mode 100644 index 000000000..01c7a4a62 --- /dev/null +++ b/packages/antlr4/src/antlr4/dfa/DFA.js @@ -0,0 +1,159 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import DFAState from './DFAState.js'; +import StarLoopEntryState from '../state/StarLoopEntryState.js'; +import ATNConfigSet from './../atn/ATNConfigSet.js'; +import DFASerializer from './DFASerializer.js'; +import LexerDFASerializer from './LexerDFASerializer.js'; +import HashSet from "../misc/HashSet.js"; + +export default class DFA { + constructor(atnStartState, decision) { + if (decision === undefined) { + decision = 0; + } + /** + * From which ATN state did we create this DFA? + */ + this.atnStartState = atnStartState; + this.decision = decision; + /** + * A set of all DFA states. Use {@link Map} so we can get old state back + * ({@link Set} only allows you to see if it's there). + */ + this._states = new HashSet(); + this.s0 = null; + /** + * {@code true} if this DFA is for a precedence decision; otherwise, + * {@code false}. This is the backing field for {@link //isPrecedenceDfa}, + * {@link //setPrecedenceDfa} + */ + this.precedenceDfa = false; + if (atnStartState instanceof StarLoopEntryState) + { + if (atnStartState.isPrecedenceDecision) { + this.precedenceDfa = true; + const precedenceState = new DFAState(null, new ATNConfigSet()); + precedenceState.edges = []; + precedenceState.isAcceptState = false; + precedenceState.requiresFullContext = false; + this.s0 = precedenceState; + } + } + } + + /** + * Get the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @return The start state corresponding to the specified precedence, or + * {@code null} if no start state exists for the specified precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see //isPrecedenceDfa() + */ + getPrecedenceStartState(precedence) { + if (!(this.precedenceDfa)) { + throw ("Only precedence DFAs may contain a precedence start state."); + } + // s0.edges is never null for a precedence DFA + if (precedence < 0 || precedence >= this.s0.edges.length) { + return null; + } + return this.s0.edges[precedence] || null; + } + + /** + * Set the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @param startState The start state corresponding to the specified + * precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see //isPrecedenceDfa() + */ + setPrecedenceStartState(precedence, startState) { + if (!(this.precedenceDfa)) { + throw ("Only precedence DFAs may contain a precedence start state."); + } + if (precedence < 0) { + return; + } + + /** + * synchronization on s0 here is ok. when the DFA is turned into a + * precedence DFA, s0 will be initialized once and not updated again + * s0.edges is never null for a precedence DFA + */ + this.s0.edges[precedence] = startState; + } + + /** + * Sets whether this is a precedence DFA. If the specified value differs + * from the current DFA configuration, the following actions are taken; + * otherwise no changes are made to the current DFA. + * + *
I use a set of ATNConfig objects not simple states. An ATNConfig + * is both a state (ala normal conversion) and a RuleContext describing + * the chain of rules (if any) followed to arrive at that state.
+ * + *A DFA state may have multiple references to a particular state, + * but with different ATN contexts (with same or different alts) + * meaning that state was reached via a different set of rule invocations.
+ */ +export default class DFAState { + constructor(stateNumber, configs) { + if (stateNumber === null) { + stateNumber = -1; + } + if (configs === null) { + configs = new ATNConfigSet(); + } + this.stateNumber = stateNumber; + this.configs = configs; + /** + * {@code edges[symbol]} points to target of symbol. Shift up by 1 so (-1) + * {@link Token//EOF} maps to {@code edges[0]}. + */ + this.edges = null; + this.isAcceptState = false; + /** + * if accept state, what ttype do we match or alt do we predict? + * This is set to {@link ATN//INVALID_ALT_NUMBER} when {@link//predicates} + * {@code !=null} or {@link //requiresFullContext}. + */ + this.prediction = 0; + this.lexerActionExecutor = null; + /** + * Indicates that this state was created during SLL prediction that + * discovered a conflict between the configurations in the state. Future + * {@link ParserATNSimulator//execATN} invocations immediately jumped doing + * full context prediction if this field is true. + */ + this.requiresFullContext = false; + /** + * During SLL parsing, this is a list of predicates associated with the + * ATN configurations of the DFA state. When we have predicates, + * {@link //requiresFullContext} is {@code false} since full context + * prediction evaluates predicates + * on-the-fly. If this is not null, then {@link //prediction} is + * {@link ATN//INVALID_ALT_NUMBER}. + * + *We only use these for non-{@link //requiresFullContext} but + * conflicting states. That + * means we know from the context (it's $ or we don't dip into outer + * context) that it's an ambiguity not a conflict.
+ * + *This list is computed by {@link + * ParserATNSimulator//predicateDFAState}.
+ */ + this.predicates = null; + return this; + } + + /** + * Get the set of all alts mentioned by all ATN configurations in this + * DFA state. + */ + getAltSet() { + const alts = new HashSet(); + if (this.configs !== null) { + for (let i = 0; i < this.configs.length; i++) { + const c = this.configs[i]; + alts.add(c.alt); + } + } + if (alts.length === 0) { + return null; + } else { + return alts; + } + } + + /** + * Two {@link DFAState} instances are equal if their ATN configuration sets + * are the same. This method is used to see if a state already exists. + * + *Because the number of alternatives and number of ATN configurations are + * finite, there is a finite number of DFA states that can be processed. + * This is necessary to show that the algorithm terminates.
+ * + *Cannot test the DFA state numbers here because in + * {@link ParserATNSimulator//addDFAState} we need to know if any other state + * exists that has this exact set of ATN configurations. The + * {@link //stateNumber} is irrelevant.
+ */ + equals(other) { + // compare set of ATN configurations in this set with other + return this === other || + (other instanceof DFAState && + this.configs.equals(other.configs)); + } + + toString() { + let s = "" + this.stateNumber + ":" + this.configs; + if(this.isAcceptState) { + s = s + "=>"; + if (this.predicates !== null) + s = s + this.predicates; + else + s = s + this.prediction; + } + return s; + } + + hashCode() { + const hash = new HashCode(); + hash.update(this.configs); + return hash.finish(); + } +} diff --git a/packages/antlr4/src/antlr4/dfa/LexerDFASerializer.js b/packages/antlr4/src/antlr4/dfa/LexerDFASerializer.js new file mode 100644 index 000000000..89da71c60 --- /dev/null +++ b/packages/antlr4/src/antlr4/dfa/LexerDFASerializer.js @@ -0,0 +1,15 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import DFASerializer from "./DFASerializer.js"; + +export default class LexerDFASerializer extends DFASerializer { + constructor(dfa) { + super(dfa, null); + } + + getEdgeLabel(i) { + return "'" + String.fromCharCode(i) + "'"; + } +} diff --git a/packages/antlr4/src/antlr4/dfa/PredPrediction.js b/packages/antlr4/src/antlr4/dfa/PredPrediction.js new file mode 100644 index 000000000..f61da1a0f --- /dev/null +++ b/packages/antlr4/src/antlr4/dfa/PredPrediction.js @@ -0,0 +1,17 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +/** + * Map a predicate to a predicted alternative. + */ +export default class PredPrediction { + constructor(pred, alt) { + this.alt = alt; + this.pred = pred; + } + + toString() { + return "(" + this.pred + ", " + this.alt + ")"; + } +} diff --git a/packages/antlr4/src/antlr4/dfa/index.d.ts b/packages/antlr4/src/antlr4/dfa/index.d.ts new file mode 100644 index 000000000..f18e75bfd --- /dev/null +++ b/packages/antlr4/src/antlr4/dfa/index.d.ts @@ -0,0 +1,2 @@ +export * from './DFA'; + diff --git a/packages/antlr4/src/antlr4/dfa/index.js b/packages/antlr4/src/antlr4/dfa/index.js new file mode 100644 index 000000000..a5075915c --- /dev/null +++ b/packages/antlr4/src/antlr4/dfa/index.js @@ -0,0 +1,11 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import DFA from './DFA.js'; +import DFASerializer from './DFASerializer.js'; +import LexerDFASerializer from './LexerDFASerializer.js'; +import PredPrediction from './PredPrediction.js'; + +export default { DFA, DFASerializer, LexerDFASerializer, PredPrediction }; diff --git a/packages/antlr4/src/antlr4/error/BailErrorStrategy.d.ts b/packages/antlr4/src/antlr4/error/BailErrorStrategy.d.ts new file mode 100644 index 000000000..c5683f658 --- /dev/null +++ b/packages/antlr4/src/antlr4/error/BailErrorStrategy.d.ts @@ -0,0 +1,7 @@ +import {DefaultErrorStrategy} from "./DefaultErrorStrategy"; + +export declare class BailErrorStrategy extends DefaultErrorStrategy { + + constructor(); + +} diff --git a/packages/antlr4/src/antlr4/error/BailErrorStrategy.js b/packages/antlr4/src/antlr4/error/BailErrorStrategy.js new file mode 100644 index 000000000..f46826755 --- /dev/null +++ b/packages/antlr4/src/antlr4/error/BailErrorStrategy.js @@ -0,0 +1,70 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import InputMismatchException from "./InputMismatchException.js"; +import ParseCancellationException from "./ParseCancellationException.js"; +import DefaultErrorStrategy from "./DefaultErrorStrategy.js"; + +/** + * This implementation of {@link ANTLRErrorStrategy} responds to syntax errors + * by immediately canceling the parse operation with a + * {@link ParseCancellationException}. The implementation ensures that the + * {@link ParserRuleContext//exception} field is set for all parse tree nodes + * that were not completed prior to encountering the error. + * + *+ * This error strategy is useful in the following scenarios.
+ * + *+ * {@code myparser.setErrorHandler(new BailErrorStrategy());}
+ * + * @see Parser//setErrorHandler(ANTLRErrorStrategy) + * */ +export default class BailErrorStrategy extends DefaultErrorStrategy { + + constructor() { + super(); + } + + /** + * Instead of recovering from exception {@code e}, re-throw it wrapped + * in a {@link ParseCancellationException} so it is not caught by the + * rule function catches. Use {@link Exception//getCause()} to get the + * original {@link RecognitionException}. + */ + recover(recognizer, e) { + let context = recognizer._ctx + while (context !== null) { + context.exception = e; + context = context.parentCtx; + } + throw new ParseCancellationException(e); + } + + /** + * Make sure we don't attempt to recover inline; if the parser + * successfully recovers, it won't throw an exception. + */ + recoverInline(recognizer) { + this.recover(recognizer, new InputMismatchException(recognizer)); + } + +// Make sure we don't attempt to recover from problems in subrules.// + sync(recognizer) { + // pass + } +} diff --git a/packages/antlr4/src/antlr4/error/ConsoleErrorListener.js b/packages/antlr4/src/antlr4/error/ConsoleErrorListener.js new file mode 100644 index 000000000..91912d0bf --- /dev/null +++ b/packages/antlr4/src/antlr4/error/ConsoleErrorListener.js @@ -0,0 +1,34 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import ErrorListener from "./ErrorListener.js"; + +/** + * {@inheritDoc} + * + *+ * This implementation prints messages to {@link System//err} containing the + * values of {@code line}, {@code charPositionInLine}, and {@code msg} using + * the following format.
+ * + *+ * line line:charPositionInLine msg + *+ * + */ +export default class ConsoleErrorListener extends ErrorListener { + constructor() { + super(); + } + + syntaxError(recognizer, offendingSymbol, line, column, msg, e) { + console.error("line " + line + ":" + column + " " + msg); + } +} + + +/** + * Provides a default instance of {@link ConsoleErrorListener}. + */ +ConsoleErrorListener.INSTANCE = new ConsoleErrorListener(); diff --git a/packages/antlr4/src/antlr4/error/DefaultErrorStrategy.d.ts b/packages/antlr4/src/antlr4/error/DefaultErrorStrategy.d.ts new file mode 100644 index 000000000..a67795bc1 --- /dev/null +++ b/packages/antlr4/src/antlr4/error/DefaultErrorStrategy.d.ts @@ -0,0 +1,24 @@ +import {ErrorStrategy} from "./ErrorStrategy"; +import {RecognitionException} from "./RecognitionException"; +import {Parser} from "../Parser"; +import {Token} from "../Token"; + +export declare class DefaultErrorStrategy implements ErrorStrategy { + recover(recognizer: Parser, e: RecognitionException): void; + + recoverInline(recognizer: Parser): Token; + + reportError(recognizer: Parser, e: RecognitionException): void; + + reportMatch(recognizer: Parser): void; + + reset(recognizer: Parser): void; + + sync(recognizer: Parser): void; + + inErrorRecoveryMode(recognizer: Parser): boolean; + + beginErrorCondition(recognizer: Parser): void; + + getMissingSymbol(recognizer: Parser): Token; +} diff --git a/packages/antlr4/src/antlr4/error/DefaultErrorStrategy.js b/packages/antlr4/src/antlr4/error/DefaultErrorStrategy.js new file mode 100644 index 000000000..f498091b8 --- /dev/null +++ b/packages/antlr4/src/antlr4/error/DefaultErrorStrategy.js @@ -0,0 +1,694 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import FailedPredicateException from "./FailedPredicateException.js"; +import InputMismatchException from "./InputMismatchException.js"; +import NoViableAltException from "./NoViableAltException.js"; +import ATNState from "../state/ATNState.js"; +import Token from '../Token.js'; +import Interval from "../misc/Interval.js"; +import IntervalSet from "../misc/IntervalSet.js"; +import ErrorStrategy from "./ErrorStrategy.js"; + +/** + * This is the default implementation of {@link ANTLRErrorStrategy} used for + * error reporting and recovery in ANTLR parsers. + */ +export default class DefaultErrorStrategy extends ErrorStrategy { + constructor() { + super(); + /** + * Indicates whether the error strategy is currently "recovering from an + * error". This is used to suppress reporting multiple error messages while + * attempting to recover from a detected syntax error. + * + * @see //inErrorRecoveryMode + */ + this.errorRecoveryMode = false; + + /** + * The index into the input stream where the last error occurred. + * This is used to prevent infinite loops where an error is found + * but no token is consumed during recovery...another error is found, + * ad nauseum. This is a failsafe mechanism to guarantee that at least + * one token/tree node is consumed for two errors. + */ + this.lastErrorIndex = -1; + this.lastErrorStates = null; + this.nextTokensContext = null; + this.nextTokenState = 0; + } + + /** + *
The default implementation simply calls {@link //endErrorCondition} to + * ensure that the handler is not in error recovery mode.
+ */ + reset(recognizer) { + this.endErrorCondition(recognizer); + } + + /** + * This method is called to enter error recovery mode when a recognition + * exception is reported. + * + * @param recognizer the parser instance + */ + beginErrorCondition(recognizer) { + this.errorRecoveryMode = true; + } + + inErrorRecoveryMode(recognizer) { + return this.errorRecoveryMode; + } + + /** + * This method is called to leave error recovery mode after recovering from + * a recognition exception. + * @param recognizer + */ + endErrorCondition(recognizer) { + this.errorRecoveryMode = false; + this.lastErrorStates = null; + this.lastErrorIndex = -1; + } + + /** + * {@inheritDoc} + *The default implementation simply calls {@link //endErrorCondition}.
+ */ + reportMatch(recognizer) { + this.endErrorCondition(recognizer); + } + + /** + * {@inheritDoc} + * + *The default implementation returns immediately if the handler is already + * in error recovery mode. Otherwise, it calls {@link //beginErrorCondition} + * and dispatches the reporting task based on the runtime type of {@code e} + * according to the following table.
+ * + *The default implementation resynchronizes the parser by consuming tokens + * until we find one in the resynchronization set--loosely the set of tokens + * that can follow the current rule.
+ * + */ + recover(recognizer, e) { + if (this.lastErrorIndex===recognizer.getInputStream().index && + this.lastErrorStates !== null && this.lastErrorStates.indexOf(recognizer.state)>=0) { + // uh oh, another error at same token index and previously-visited + // state in ATN; must be a case where LT(1) is in the recovery + // token set so nothing got consumed. Consume a single token + // at least to prevent an infinite loop; this is a failsafe. + recognizer.consume(); + } + this.lastErrorIndex = recognizer._input.index; + if (this.lastErrorStates === null) { + this.lastErrorStates = []; + } + this.lastErrorStates.push(recognizer.state); + const followSet = this.getErrorRecoverySet(recognizer) + this.consumeUntil(recognizer, followSet); + } + + /** + * The default implementation of {@link ANTLRErrorStrategy//sync} makes sure + * that the current lookahead symbol is consistent with what were expecting + * at this point in the ATN. You can call this anytime but ANTLR only + * generates code to check before subrules/loops and each iteration. + * + *Implements Jim Idle's magic sync mechanism in closures and optional + * subrules. E.g.,
+ * + *
+ * a : sync ( stuff sync )* ;
+ * sync : {consume to what can follow sync} ;
+ *
+ *
+ * At the start of a sub rule upon error, {@link //sync} performs single
+ * token deletion, if possible. If it can't do that, it bails on the current
+ * rule and uses the default error recovery, which consumes until the
+ * resynchronization set of the current rule.
+ *
+ * If the sub rule is optional ({@code (...)?}, {@code (...)*}, or block + * with an empty alternative), then the expected set includes what follows + * the subrule.
+ * + *During loop iteration, it consumes until it sees a token that can start a + * sub rule or what follows loop. Yes, that is pretty aggressive. We opt to + * stay in the loop as long as possible.
+ * + *ORIGINS
+ * + *Previous versions of ANTLR did a poor job of their recovery within loops. + * A single mismatch token or missing token would force the parser to bail + * out of the entire rules surrounding the loop. So, for rule
+ * + *
+ * classDef : 'class' ID '{' member* '}'
+ *
+ *
+ * input with an extra token between members would force the parser to
+ * consume until it found the next class definition rather than the next
+ * member definition of the current class.
+ *
+ * This functionality cost a little bit of effort because the parser has to + * compare token set at the start of the loop and at each iteration. If for + * some reason speed is suffering for you, you can turn off this + * functionality by simply overriding this method as a blank { }.
+ * + */ + sync(recognizer) { + // If already recovering, don't try to sync + if (this.inErrorRecoveryMode(recognizer)) { + return; + } + const s = recognizer._interp.atn.states[recognizer.state]; + const la = recognizer.getTokenStream().LA(1); + // try cheaper subset first; might get lucky. seems to shave a wee bit off + const nextTokens = recognizer.atn.nextTokens(s); + if(nextTokens.contains(la)) { + this.nextTokensContext = null; + this.nextTokenState = ATNState.INVALID_STATE_NUMBER; + return; + } else if (nextTokens.contains(Token.EPSILON)) { + if(this.nextTokensContext === null) { + // It's possible the next token won't match information tracked + // by sync is restricted for performance. + this.nextTokensContext = recognizer._ctx; + this.nextTokensState = recognizer._stateNumber; + } + return; + } + switch (s.stateType) { + case ATNState.BLOCK_START: + case ATNState.STAR_BLOCK_START: + case ATNState.PLUS_BLOCK_START: + case ATNState.STAR_LOOP_ENTRY: + // report error and recover if possible + if( this.singleTokenDeletion(recognizer) !== null) { + return; + } else { + throw new InputMismatchException(recognizer); + } + case ATNState.PLUS_LOOP_BACK: + case ATNState.STAR_LOOP_BACK: + { + this.reportUnwantedToken(recognizer); + const expecting = new IntervalSet(); + expecting.addSet(recognizer.getExpectedTokens()); + const whatFollowsLoopIterationOrRule = expecting.addSet(this.getErrorRecoverySet(recognizer)); + this.consumeUntil(recognizer, whatFollowsLoopIterationOrRule); + } + break; + default: + // do nothing if we can't identify the exact kind of ATN state + } + } + + /** + * This is called by {@link //reportError} when the exception is a + * {@link NoViableAltException}. + * + * @see //reportError + * + * @param recognizer the parser instance + * @param e the recognition exception + */ + reportNoViableAlternative(recognizer, e) { + const tokens = recognizer.getTokenStream() + let input + if(tokens !== null) { + if (e.startToken.type===Token.EOF) { + input = "This method is called when {@link //singleTokenDeletion} identifies + * single-token deletion as a viable recovery strategy for a mismatched + * input error.
+ * + *The default implementation simply returns if the handler is already in + * error recovery mode. Otherwise, it calls {@link //beginErrorCondition} to + * enter error recovery mode, followed by calling + * {@link Parser//notifyErrorListeners}.
+ * + * @param recognizer the parser instance + * + */ + reportUnwantedToken(recognizer) { + if (this.inErrorRecoveryMode(recognizer)) { + return; + } + this.beginErrorCondition(recognizer); + const t = recognizer.getCurrentToken() + const tokenName = this.getTokenErrorDisplay(t) + const expecting = this.getExpectedTokens(recognizer) + const msg = "extraneous input " + tokenName + " expecting " + + expecting.toString(recognizer.literalNames, recognizer.symbolicNames) + recognizer.notifyErrorListeners(msg, t, null); + } + + /** + * This method is called to report a syntax error which requires the + * insertion of a missing token into the input stream. At the time this + * method is called, the missing token has not yet been inserted. When this + * method returns, {@code recognizer} is in error recovery mode. + * + *This method is called when {@link //singleTokenInsertion} identifies + * single-token insertion as a viable recovery strategy for a mismatched + * input error.
+ * + *The default implementation simply returns if the handler is already in + * error recovery mode. Otherwise, it calls {@link //beginErrorCondition} to + * enter error recovery mode, followed by calling + * {@link Parser//notifyErrorListeners}.
+ * + * @param recognizer the parser instance + */ + reportMissingToken(recognizer) { + if ( this.inErrorRecoveryMode(recognizer)) { + return; + } + this.beginErrorCondition(recognizer); + const t = recognizer.getCurrentToken() + const expecting = this.getExpectedTokens(recognizer) + const msg = "missing " + expecting.toString(recognizer.literalNames, recognizer.symbolicNames) + + " at " + this.getTokenErrorDisplay(t) + recognizer.notifyErrorListeners(msg, t, null); + } + + /** + *The default implementation attempts to recover from the mismatched input + * by using single token insertion and deletion as described below. If the + * recovery attempt fails, this method throws an + * {@link InputMismatchException}.
+ * + *EXTRA TOKEN (single token deletion)
+ * + *{@code LA(1)} is not what we are looking for. If {@code LA(2)} has the + * right token, however, then assume {@code LA(1)} is some extra spurious + * token and delete it. Then consume and return the next token (which was + * the {@code LA(2)} token) as the successful result of the match operation.
+ * + *This recovery strategy is implemented by {@link + * //singleTokenDeletion}.
+ * + *MISSING TOKEN (single token insertion)
+ * + *If current token (at {@code LA(1)}) is consistent with what could come + * after the expected {@code LA(1)} token, then assume the token is missing + * and use the parser's {@link TokenFactory} to create it on the fly. The + * "insertion" is performed by returning the created token as the successful + * result of the match operation.
+ * + *This recovery strategy is implemented by {@link + * //singleTokenInsertion}.
+ * + *EXAMPLE
+ * + *For example, Input {@code i=(3;} is clearly missing the {@code ')'}. When + * the parser returns from the nested call to {@code expr}, it will have + * call chain:
+ * + *+ * stat → expr → atom + *+ * + * and it will be trying to match the {@code ')'} at this point in the + * derivation: + * + *
+ * => ID '=' '(' INT ')' ('+' atom)* ';'
+ * ^
+ *
+ *
+ * The attempt to match {@code ')'} will fail when it sees {@code ';'} and
+ * call {@link //recoverInline}. To recover, it sees that {@code LA(1)==';'}
+ * is in the set of tokens that can follow the {@code ')'} token reference
+ * in rule {@code atom}. It can assume that you forgot the {@code ')'}.
+ */
+ recoverInline(recognizer) {
+ // SINGLE TOKEN DELETION
+ const matchedSymbol = this.singleTokenDeletion(recognizer)
+ if (matchedSymbol !== null) {
+ // we have deleted the extra token.
+ // now, move past ttype token as if all were ok
+ recognizer.consume();
+ return matchedSymbol;
+ }
+ // SINGLE TOKEN INSERTION
+ if (this.singleTokenInsertion(recognizer)) {
+ return this.getMissingSymbol(recognizer);
+ }
+ // even that didn't work; must throw the exception
+ throw new InputMismatchException(recognizer);
+ }
+
+ /**
+ * This method implements the single-token insertion inline error recovery
+ * strategy. It is called by {@link //recoverInline} if the single-token
+ * deletion strategy fails to recover from the mismatched input. If this
+ * method returns {@code true}, {@code recognizer} will be in error recovery
+ * mode.
+ *
+ * This method determines whether or not single-token insertion is viable by + * checking if the {@code LA(1)} input symbol could be successfully matched + * if it were instead the {@code LA(2)} symbol. If this method returns + * {@code true}, the caller is responsible for creating and inserting a + * token with the correct type to produce this behavior.
+ * + * @param recognizer the parser instance + * @return {@code true} if single-token insertion is a viable recovery + * strategy for the current mismatched input, otherwise {@code false} + */ + singleTokenInsertion(recognizer) { + const currentSymbolType = recognizer.getTokenStream().LA(1) + // if current token is consistent with what could come after current + // ATN state, then we know we're missing a token; error recovery + // is free to conjure up and insert the missing token + const atn = recognizer._interp.atn + const currentState = atn.states[recognizer.state] + const next = currentState.transitions[0].target + const expectingAtLL2 = atn.nextTokens(next, recognizer._ctx) + if (expectingAtLL2.contains(currentSymbolType) ){ + this.reportMissingToken(recognizer); + return true; + } else { + return false; + } + } + + /** + * This method implements the single-token deletion inline error recovery + * strategy. It is called by {@link //recoverInline} to attempt to recover + * from mismatched input. If this method returns null, the parser and error + * handler state will not have changed. If this method returns non-null, + * {@code recognizer} will not be in error recovery mode since the + * returned token was a successful match. + * + *If the single-token deletion is successful, this method calls + * {@link //reportUnwantedToken} to report the error, followed by + * {@link Parser//consume} to actually "delete" the extraneous token. Then, + * before returning {@link //reportMatch} is called to signal a successful + * match.
+ * + * @param recognizer the parser instance + * @return the successfully matched {@link Token} instance if single-token + * deletion successfully recovers from the mismatched input, otherwise + * {@code null} + */ + singleTokenDeletion(recognizer) { + const nextTokenType = recognizer.getTokenStream().LA(2) + const expecting = this.getExpectedTokens(recognizer) + if (expecting.contains(nextTokenType)) { + this.reportUnwantedToken(recognizer); + // print("recoverFromMismatchedToken deleting " \ + // + str(recognizer.getTokenStream().LT(1)) \ + // + " since " + str(recognizer.getTokenStream().LT(2)) \ + // + " is what we want", file=sys.stderr) + recognizer.consume(); // simply delete extra token + // we want to return the token we're actually matching + const matchedSymbol = recognizer.getCurrentToken() + this.reportMatch(recognizer); // we know current token is correct + return matchedSymbol; + } else { + return null; + } + } + + /** + * Conjure up a missing token during error recovery. + * + * The recognizer attempts to recover from single missing + * symbols. But, actions might refer to that missing symbol. + * For example, x=ID {f($x);}. The action clearly assumes + * that there has been an identifier matched previously and that + * $x points at that token. If that token is missing, but + * the next token in the stream is what we want we assume that + * this token is missing and we keep going. Because we + * have to return some token to replace the missing token, + * we have to conjure one up. This method gives the user control + * over the tokens returned for missing tokens. Mostly, + * you will want to create something special for identifier + * tokens. For literals such as '{' and ',', the default + * action in the parser or tree parser works. It simply creates + * a CommonToken of the appropriate type. The text will be the token. + * If you change what tokens must be created by the lexer, + * override this method to create the appropriate tokens. + * + */ + getMissingSymbol(recognizer) { + const currentSymbol = recognizer.getCurrentToken() + const expecting = this.getExpectedTokens(recognizer) + const expectedTokenType = expecting.first() // get any element + let tokenText + if (expectedTokenType===Token.EOF) { + tokenText = "If the set of expected tokens is not known and could not be computed, + * this method returns {@code null}.
+ * + * @return The set of token types that could potentially follow the current + * state in the ATN, or {@code null} if the information is not available. + */ + getExpectedTokens() { + if (this.recognizer!==null) { + return this.recognizer.atn.getExpectedTokens(this.offendingState, this.ctx); + } else { + return null; + } + } + + //If the state number is not known, this method returns -1.
+ toString() { + return this.message; + } +} + + + diff --git a/packages/antlr4/src/antlr4/error/index.d.ts b/packages/antlr4/src/antlr4/error/index.d.ts new file mode 100644 index 000000000..2af8a0c5f --- /dev/null +++ b/packages/antlr4/src/antlr4/error/index.d.ts @@ -0,0 +1,9 @@ +export * from './RecognitionException'; +export * from './NoViableAltException'; +export * from './FailedPredicateException'; +export * from './InputMismatchException'; +export * from './ErrorStrategy'; +export * from './BailErrorStrategy'; +export * from './DefaultErrorStrategy'; +export * from './ErrorListener'; +export * from './DiagnosticErrorListener'; diff --git a/packages/antlr4/src/antlr4/error/index.js b/packages/antlr4/src/antlr4/error/index.js new file mode 100644 index 000000000..d41bd9125 --- /dev/null +++ b/packages/antlr4/src/antlr4/error/index.js @@ -0,0 +1,19 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import RecognitionException from './RecognitionException.js'; +import NoViableAltException from './NoViableAltException.js'; +import LexerNoViableAltException from './LexerNoViableAltException.js'; +import InputMismatchException from './InputMismatchException.js'; +import FailedPredicateException from './FailedPredicateException.js'; +import DiagnosticErrorListener from './DiagnosticErrorListener.js'; +import BailErrorStrategy from './BailErrorStrategy.js'; +import DefaultErrorStrategy from './DefaultErrorStrategy.js'; +import ErrorListener from './ErrorListener.js'; + +export default { + RecognitionException, NoViableAltException, LexerNoViableAltException, InputMismatchException, FailedPredicateException, + DiagnosticErrorListener, BailErrorStrategy, DefaultErrorStrategy, ErrorListener +} diff --git a/packages/antlr4/src/antlr4/index.d.ts b/packages/antlr4/src/antlr4/index.d.ts new file mode 100644 index 000000000..3c6c9bcf8 --- /dev/null +++ b/packages/antlr4/src/antlr4/index.d.ts @@ -0,0 +1,21 @@ +export * from "./InputStream"; +export * from "./FileStream"; +export * from "./CharStream"; +export * from "./CharStreams"; +export * from "./TokenStream"; +export * from "./BufferedTokenStream"; +export * from "./CommonToken"; +export * from "./CommonTokenStream"; +export * from "./Recognizer"; +export * from "./Lexer"; +export * from "./Parser"; +export * from './Token'; +export * from "./atn"; +export * from "./dfa"; +export * from "./context"; +export * from './misc'; +export * from './tree'; +export * from './state'; +export * from './error'; +export * from './utils'; +export * from './TokenStreamRewriter'; diff --git a/packages/antlr4/src/antlr4/index.node.js b/packages/antlr4/src/antlr4/index.node.js new file mode 100644 index 000000000..b54aa2ccd --- /dev/null +++ b/packages/antlr4/src/antlr4/index.node.js @@ -0,0 +1,61 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import { default as atn } from './atn/index.js'; +import { default as dfa } from './dfa/index.js'; +import { default as context } from './context/index.js'; +import { default as misc } from './misc/index.js'; +import { default as tree } from './tree/index.js'; +import { default as error } from './error/index.js'; +import { default as CharStreams } from './CharStreams.js'; +import { default as Utils } from './utils/index.js'; + +import Token from './Token.js'; +import CommonToken from './CommonToken.js'; +import InputStream from './InputStream.js'; +import CharStream from './InputStream.js'; +import FileStream from './FileStream.js'; +import CommonTokenStream from './CommonTokenStream.js'; +import Lexer from './Lexer.js'; +import Parser from './Parser.js'; + +import RuleContext from './context/RuleContext.js'; +import ParserRuleContext from './context/ParserRuleContext.js'; +import ATN from './atn/ATN.js'; +import PredictionMode from './atn/PredictionMode.js'; +import LL1Analyzer from './atn/LL1Analyzer.js'; +import ATNDeserializer from './atn/ATNDeserializer.js'; +import LexerATNSimulator from './atn/LexerATNSimulator.js'; +import ParserATNSimulator from './atn/ParserATNSimulator.js'; +import PredictionContextCache from './atn/PredictionContextCache.js'; +import DFA from "./dfa/DFA.js"; +import RecognitionException from "./error/RecognitionException.js"; +import FailedPredicateException from "./error/FailedPredicateException.js"; +import NoViableAltException from "./error/NoViableAltException.js"; +import BailErrorStrategy from "./error/BailErrorStrategy.js"; +import DefaultErrorStrategy from "./error/DefaultErrorStrategy.js"; +import Interval from './misc/Interval.js'; +import IntervalSet from './misc/IntervalSet.js'; +import ParseTreeListener from "./tree/ParseTreeListener.js"; +import ParseTreeVisitor from "./tree/ParseTreeVisitor.js"; +import ParseTreeWalker from "./tree/ParseTreeWalker.js"; +import ErrorListener from "./error/ErrorListener.js" +import DiagnosticErrorListener from "./error/DiagnosticErrorListener.js" +import RuleNode from "./tree/RuleNode.js" +import TerminalNode from "./tree/TerminalNode.js" +import arrayToString from "./utils/arrayToString.js" +import TokenStreamRewriter from './TokenStreamRewriter.js'; + +export default { + atn, dfa, context, misc, tree, error, Token, CommonToken, CharStreams, CharStream, InputStream, FileStream, CommonTokenStream, Lexer, Parser, + ParserRuleContext, Interval, IntervalSet, LL1Analyzer, Utils, TokenStreamRewriter +} + +export { + Token, CommonToken, CharStreams, CharStream, InputStream, FileStream, CommonTokenStream, Lexer, Parser, + RuleNode, TerminalNode, ParseTreeWalker, RuleContext, ParserRuleContext, Interval, IntervalSet, + PredictionMode, LL1Analyzer, ParseTreeListener, ParseTreeVisitor, ATN, ATNDeserializer, PredictionContextCache, LexerATNSimulator, ParserATNSimulator, DFA, + RecognitionException, NoViableAltException, FailedPredicateException, ErrorListener, DiagnosticErrorListener, BailErrorStrategy, DefaultErrorStrategy, + arrayToString +} diff --git a/packages/antlr4/src/antlr4/index.web.js b/packages/antlr4/src/antlr4/index.web.js new file mode 100644 index 000000000..6361803b1 --- /dev/null +++ b/packages/antlr4/src/antlr4/index.web.js @@ -0,0 +1,60 @@ +/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import { default as atn } from './atn/index.js'; +import { default as dfa } from './dfa/index.js'; +import { default as context } from './context/index.js'; +import { default as misc } from './misc/index.js'; +import { default as tree } from './tree/index.js'; +import { default as error } from './error/index.js'; +import { default as CharStreams } from './CharStreams.js'; +import { default as Utils } from './utils/index.js'; + +import Token from './Token.js'; +import CommonToken from './CommonToken.js'; +import InputStream from './InputStream.js'; +import CharStream from './InputStream.js'; +import CommonTokenStream from './CommonTokenStream.js'; +import Lexer from './Lexer.js'; +import Parser from './Parser.js'; + +import RuleContext from './context/RuleContext.js'; +import ParserRuleContext from './context/ParserRuleContext.js'; +import ATN from './atn/ATN.js'; +import PredictionMode from './atn/PredictionMode.js'; +import LL1Analyzer from './atn/LL1Analyzer.js'; +import ATNDeserializer from './atn/ATNDeserializer.js'; +import LexerATNSimulator from './atn/LexerATNSimulator.js'; +import ParserATNSimulator from './atn/ParserATNSimulator.js'; +import PredictionContextCache from './atn/PredictionContextCache.js'; +import DFA from "./dfa/DFA.js"; +import RecognitionException from "./error/RecognitionException.js"; +import FailedPredicateException from "./error/FailedPredicateException.js"; +import NoViableAltException from "./error/NoViableAltException.js"; +import BailErrorStrategy from "./error/BailErrorStrategy.js"; +import DefaultErrorStrategy from "./error/DefaultErrorStrategy.js"; +import Interval from './misc/Interval.js'; +import IntervalSet from './misc/IntervalSet.js'; +import ParseTreeListener from "./tree/ParseTreeListener.js"; +import ParseTreeVisitor from "./tree/ParseTreeVisitor.js"; +import ParseTreeWalker from "./tree/ParseTreeWalker.js"; +import ErrorListener from "./error/ErrorListener.js" +import DiagnosticErrorListener from "./error/DiagnosticErrorListener.js" +import RuleNode from "./tree/RuleNode.js" +import TerminalNode from "./tree/TerminalNode.js" +import arrayToString from "./utils/arrayToString.js" +import TokenStreamRewriter from './TokenStreamRewriter.js'; + +export default { + atn, dfa, context, misc, tree, error, Token, CommonToken, CharStreams, CharStream, InputStream, CommonTokenStream, Lexer, Parser, + ParserRuleContext, Interval, IntervalSet, LL1Analyzer, Utils, TokenStreamRewriter +} + +export { + Token, CommonToken, CharStreams, CharStream, InputStream, CommonTokenStream, Lexer, Parser, + RuleNode, TerminalNode, ParseTreeWalker, RuleContext, ParserRuleContext, Interval, IntervalSet, + PredictionMode, LL1Analyzer, ParseTreeListener, ParseTreeVisitor, ATN, ATNDeserializer, PredictionContextCache, LexerATNSimulator, ParserATNSimulator, DFA, + RecognitionException, NoViableAltException, FailedPredicateException, ErrorListener, DiagnosticErrorListener, BailErrorStrategy, DefaultErrorStrategy, + arrayToString +} diff --git a/packages/antlr4/src/antlr4/misc/AltDict.js b/packages/antlr4/src/antlr4/misc/AltDict.js new file mode 100644 index 000000000..436ce2eff --- /dev/null +++ b/packages/antlr4/src/antlr4/misc/AltDict.js @@ -0,0 +1,22 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +export default class AltDict { + + constructor() { + this.data = {}; + } + + get(key) { + return this.data["k-" + key] || null; + } + + set(key, value) { + this.data["k-" + key] = value; + } + + values() { + return Object.keys(this.data).filter(key => key.startsWith("k-")).map(key => this.data[key], this); + } +} diff --git a/packages/antlr4/src/antlr4/misc/BitSet.js b/packages/antlr4/src/antlr4/misc/BitSet.js new file mode 100644 index 000000000..90df4e9c7 --- /dev/null +++ b/packages/antlr4/src/antlr4/misc/BitSet.js @@ -0,0 +1,53 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import HashCode from "./HashCode.js"; +import equalArrays from "../utils/equalArrays.js"; + +export default class BitSet { + + constructor() { + this.data = []; + } + + add(value) { + this.data[value] = true; + } + + or(set) { + Object.keys(set.data).map(alt => this.add(alt), this); + } + + remove(value) { + delete this.data[value]; + } + + has(value) { + return this.data[value] === true; + } + + values() { + return Object.keys(this.data); + } + + minValue() { + return Math.min.apply(null, this.values()); + } + + hashCode() { + return HashCode.hashStuff(this.values()); + } + + equals(other) { + return other instanceof BitSet && equalArrays(this.data, other.data); + } + + toString() { + return "{" + this.values().join(", ") + "}"; + } + + get length(){ + return this.values().length; + } +} diff --git a/packages/antlr4/src/antlr4/misc/HashCode.js b/packages/antlr4/src/antlr4/misc/HashCode.js new file mode 100644 index 000000000..1de3c6952 --- /dev/null +++ b/packages/antlr4/src/antlr4/misc/HashCode.js @@ -0,0 +1,68 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import { stringHashCode } from "../utils/stringHashCode.js"; + +export default class HashCode { + + constructor() { + this.count = 0; + this.hash = 0; + } + + update() { + for(let i=0;iThis is a one way link. It emanates from a state (usually via a list of + * transitions) and has a target state.
+ * + *Since we never have to change the ATN transitions once we construct it, + * we can fix these transitions as specific classes. The DFA transitions + * on the other hand need to update the labels as it adds transitions to + * the states. We'll use the term Edge for the DFA to distinguish them from + * ATN transitions.
+ */ +export default class Transition { + constructor(target) { + // The target of this transition. + if (target===undefined || target===null) { + throw "target cannot be null."; + } + this.target = target; + // Are we epsilon, action, sempred? + this.isEpsilon = false; + this.label = null; + } +} + +// constants for serialization + +Transition.EPSILON = 1; +Transition.RANGE = 2; +Transition.RULE = 3; +// e.g., {isType(input.LT(1))}? +Transition.PREDICATE = 4; +Transition.ATOM = 5; +Transition.ACTION = 6; +// ~(A|B) or ~atom, wildcard, which convert to next 2 +Transition.SET = 7; +Transition.NOT_SET = 8; +Transition.WILDCARD = 9; +Transition.PRECEDENCE = 10; + +Transition.serializationNames = [ + "INVALID", + "EPSILON", + "RANGE", + "RULE", + "PREDICATE", + "ATOM", + "ACTION", + "SET", + "NOT_SET", + "WILDCARD", + "PRECEDENCE" + ]; + +Transition.serializationTypes = { + EpsilonTransition: Transition.EPSILON, + RangeTransition: Transition.RANGE, + RuleTransition: Transition.RULE, + PredicateTransition: Transition.PREDICATE, + AtomTransition: Transition.ATOM, + ActionTransition: Transition.ACTION, + SetTransition: Transition.SET, + NotSetTransition: Transition.NOT_SET, + WildcardTransition: Transition.WILDCARD, + PrecedencePredicateTransition: Transition.PRECEDENCE + }; + diff --git a/packages/antlr4/src/antlr4/transition/WildcardTransition.js b/packages/antlr4/src/antlr4/transition/WildcardTransition.js new file mode 100644 index 000000000..ba8ef0897 --- /dev/null +++ b/packages/antlr4/src/antlr4/transition/WildcardTransition.js @@ -0,0 +1,20 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import Transition from "./Transition.js"; + +export default class WildcardTransition extends Transition { + constructor(target) { + super(target); + this.serializationType = Transition.WILDCARD; + } + + matches(symbol, minVocabSymbol, maxVocabSymbol) { + return symbol >= minVocabSymbol && symbol <= maxVocabSymbol; + } + + toString() { + return "."; + } +} diff --git a/packages/antlr4/src/antlr4/tree/ErrorNode.d.ts b/packages/antlr4/src/antlr4/tree/ErrorNode.d.ts new file mode 100644 index 000000000..b93cd50fe --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ErrorNode.d.ts @@ -0,0 +1,5 @@ +import {TerminalNode} from "./TerminalNode"; + +export declare class ErrorNode extends TerminalNode { + +} diff --git a/packages/antlr4/src/antlr4/tree/ErrorNode.js b/packages/antlr4/src/antlr4/tree/ErrorNode.js new file mode 100644 index 000000000..6425145ac --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ErrorNode.js @@ -0,0 +1,8 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import TerminalNode from "./TerminalNode.js"; + +export default class ErrorNode extends TerminalNode { +} diff --git a/packages/antlr4/src/antlr4/tree/ErrorNodeImpl.js b/packages/antlr4/src/antlr4/tree/ErrorNodeImpl.js new file mode 100644 index 000000000..5d9acf77d --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ErrorNodeImpl.js @@ -0,0 +1,26 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +/** + * Represents a token that was consumed during resynchronization + * rather than during a valid match operation. For example, + * we will create this kind of a node during single token insertion + * and deletion as well as during "consume until error recovery set" + * upon no viable alternative exceptions. + */ +import TerminalNodeImpl from "./TerminalNodeImpl.js"; + +export default class ErrorNodeImpl extends TerminalNodeImpl { + constructor(token) { + super(token); + } + + isErrorNode() { + return true; + } + + accept(visitor) { + return visitor.visitErrorNode(this); + } +} diff --git a/packages/antlr4/src/antlr4/tree/ParseTree.d.ts b/packages/antlr4/src/antlr4/tree/ParseTree.d.ts new file mode 100644 index 000000000..bbf86c705 --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ParseTree.d.ts @@ -0,0 +1,5 @@ +import {SyntaxTree} from "./SyntaxTree"; + +export declare class ParseTree extends SyntaxTree { + getText(): string; +} diff --git a/packages/antlr4/src/antlr4/tree/ParseTree.js b/packages/antlr4/src/antlr4/tree/ParseTree.js new file mode 100644 index 000000000..7c49fc73a --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ParseTree.js @@ -0,0 +1,8 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import SyntaxTree from "./SyntaxTree.js"; + +export default class ParseTree extends SyntaxTree { +} diff --git a/packages/antlr4/src/antlr4/tree/ParseTreeListener.d.ts b/packages/antlr4/src/antlr4/tree/ParseTreeListener.d.ts new file mode 100644 index 000000000..af2ecc974 --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ParseTreeListener.d.ts @@ -0,0 +1,10 @@ +import {ParserRuleContext} from "../context"; +import {ErrorNode} from "./ErrorNode"; +import {TerminalNode} from "./TerminalNode"; + +export declare abstract class ParseTreeListener { + visitTerminal(node: TerminalNode) : void; + visitErrorNode(node: ErrorNode) : void; + enterEveryRule(ctx: ParserRuleContext) : void; + exitEveryRule(ctx: ParserRuleContext) : void; +} diff --git a/packages/antlr4/src/antlr4/tree/ParseTreeListener.js b/packages/antlr4/src/antlr4/tree/ParseTreeListener.js new file mode 100644 index 000000000..c90a9b6d4 --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ParseTreeListener.js @@ -0,0 +1,17 @@ +/* Copyright (c) 2012-2022 The ANTLR Project Contributors. All rights reserved. + * Use is of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +export default class ParseTreeListener { + visitTerminal(node) { + } + + visitErrorNode(node) { + } + + enterEveryRule(node) { + } + + exitEveryRule(node) { + } +} diff --git a/packages/antlr4/src/antlr4/tree/ParseTreeVisitor.d.ts b/packages/antlr4/src/antlr4/tree/ParseTreeVisitor.d.ts new file mode 100644 index 000000000..9329b4913 --- /dev/null +++ b/packages/antlr4/src/antlr4/tree/ParseTreeVisitor.d.ts @@ -0,0 +1,16 @@ +import {RuleNode} from "./RuleNode"; +import {ErrorNode} from "./ErrorNode"; +import {TerminalNode} from "./TerminalNode"; +import {ParseTree} from "./ParseTree"; + +export declare class ParseTreeVisitor