diff --git a/doc/options.md b/doc/options.md index f5faa7ec37..061f38c6c8 100644 --- a/doc/options.md +++ b/doc/options.md @@ -36,6 +36,86 @@ $ antlr4 -Dlanguage=C MyGrammar.g4 error(31): ANTLR cannot generate C code as of version 4.0 ``` +### `actionTemplates` + +This option uses the provided [StringTemplate](https://www.stringtemplate.org/) group file (`*.stg`) to render templates inside the action blocks of an ANTLR grammar. + +This enables you to provide target-specific action logic by providing different `.stg` files for each target language. + +The syntax of group files is [described](https://github.com/antlr/stringtemplate4/blob/master/doc/groups.md) in the StringTemplate documentation. + +For example, if you provide the following group file when generating Java code: + +`ActionTemplates.stg`: +```string-template +normalize(s) ::= <, Form.NFKC)>> +setText(s) ::= <);>> +getText() ::= <> +normalizerImports ::= << +import java.text.Normalizer; +import java.text.Normalizer.Form; +>> +``` + +You can use the templates like so in your ANTLR grammar: + +```antlrv4 +ID: + (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { + + }; +``` + +The ANTLR tool must be invoked by providing the target language and StringTemplate group file: + +```bash +$ antlr4 -Dlanguage=Java -DactionTemplates=ActionTemplates.stg MyGrammar.g4 +``` + +The templates will be expanded into the following before the grammar is used to generate the target code: + +```antlrv4 +ID: + (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { + setText(Normalizer.normalize(getText(), Form.NFKC)); + }; +``` + +Templates can also be used in named actions, such as the `@header` or `@members` block, for example: + +```antlrv4 +@lexer::header { + +} +``` + +To use the same grammar to generate a different target language, you can provide a different StringTemplate group file. + +For example, to generate JavaScript code equivalent to the previous example the following group file could be used instead: + +`ActionTemplates.stg`: +```string-template +normalize(s) ::= <<.normalize("NFKC")>> +setText(s) ::= <;>> +getText() ::= <> +normalizerImports ::= "" +``` + +Now you can invoke the ANTLR tool with the new target language and your alternate StringTemplate group file: + +```bash +$ antlr4 -Dlanguage=JavaScript -DactionTemplates=ActionTemplates.stg MyGrammar.g4 +``` + +These templates will expand into the following before the grammar is used to generate the target code: + +```antlrv4 +ID: + (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { + this.text = this.text.normalize("NFKC"); + }; +``` + ### `tokenVocab` ANTLR assigns token type numbers to the tokens as it encounters them in a file. To use different token type values, such as with a separate lexer, use this option to have ANTLR pull in the tokens file. ANTLR generates a tokens file from each grammar. diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RunOptions.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RunOptions.java index f4f5aa869b..29fe6e4d7c 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/RunOptions.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/RunOptions.java @@ -24,6 +24,8 @@ public class RunOptions { public final boolean showDFA; public final Stage endStage; public final String superClass; + public final String libDir; + public final String actionTemplates; public final PredictionMode predictionMode; public final boolean buildParseTree; @@ -31,7 +33,7 @@ public RunOptions(String grammarFileName, String grammarStr, String parserName, boolean useListener, boolean useVisitor, String startRuleName, String input, boolean profile, boolean showDiagnosticErrors, boolean traceATN, boolean showDFA, Stage endStage, - String language, String superClass, PredictionMode predictionMode, boolean buildParseTree) { + String language, String superClass, String libDir, String actionTemplates, PredictionMode predictionMode, boolean buildParseTree) { this.grammarFileName = grammarFileName; this.grammarStr = grammarStr; this.parserName = parserName; @@ -69,6 +71,8 @@ else if (lexerName != null) { this.showDFA = showDFA; this.endStage = endStage; this.superClass = superClass; + this.libDir = libDir; + this.actionTemplates = actionTemplates; this.predictionMode = predictionMode; this.buildParseTree = buildParseTree; } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java index 46887f4494..7840f33b12 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java @@ -168,10 +168,20 @@ public State run(RunOptions runOptions) { if (runOptions.useVisitor) { options.add("-visitor"); } + + if (runOptions.libDir != null && runOptions.libDir.length() > 0) { + options.add("-lib"); + options.add(runOptions.libDir); + } + if (runOptions.superClass != null && runOptions.superClass.length() > 0) { options.add("-DsuperClass=" + runOptions.superClass); } + if (runOptions.actionTemplates != null && runOptions.actionTemplates.length() > 0) { + options.add("-DactionTemplates=" + runOptions.actionTemplates); + } + // See if the target wants to add tool options. // List targetOpts = getTargetToolOptions(runOptions); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java index 09931d0864..762514b517 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java @@ -167,6 +167,8 @@ private static String test(RuntimeTestDescriptor descriptor, RuntimeRunner runne Stage.Execute, targetName, superClass, + null, + null, descriptor.predictionMode, descriptor.buildParseTree ); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/TraceATN.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/TraceATN.java index 067f3ba1c0..4aabb9e277 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/TraceATN.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/TraceATN.java @@ -158,6 +158,8 @@ public String test(RuntimeTestDescriptor descriptor, RuntimeRunner runner, Strin Stage.Execute, targetName, superClass, + null, + null, PredictionMode.LL, true ); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java new file mode 100644 index 0000000000..981b46b4a3 --- /dev/null +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java @@ -0,0 +1,375 @@ +package org.antlr.v4.test.tool; + +import org.antlr.v4.test.runtime.RunOptions; +import org.antlr.v4.test.runtime.Stage; +import org.antlr.v4.test.runtime.java.JavaRunner; +import org.antlr.v4.test.runtime.states.ExecutedState; +import org.antlr.v4.test.runtime.states.GeneratedState; +import org.antlr.v4.test.runtime.states.State; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.antlr.v4.test.runtime.FileUtils.writeFile; +import static org.antlr.v4.test.runtime.RuntimeTestUtils.FileSeparator; +import static org.antlr.v4.test.tool.ToolTestUtils.createOptionsForJavaToolTests; +import static org.junit.jupiter.api.Assertions.*; + +public class TestActionTemplates { + @Test void testIncorrectActionTemplateGroupExtension(@TempDir Path tempDir) { + writeFile(tempDir.toString(), "Java.st", ""); + + String actionTemplates = tempDir + FileSeparator + "Java.st"; + + String grammar = + "lexer grammar L;" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(208): error reading action templates file " + actionTemplates + ": " + + "Group file names must end in .stg: " + actionTemplates + "\n", + generated.getErrorMessage()); + } + + @Test void testActionTemplateFileMissing(@TempDir Path tempDir) { + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + + String grammar = + "lexer grammar L;" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(206): cannot find action templates file " + actionTemplates + " given for L\n", + generated.getErrorMessage()); + } + + @Test void testUnlexableActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {<¢>} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 2:16: invalid character '¢'\n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 2:14: this doesn't look like a template: \" <¢> \"\n", + generated.getErrorMessage()); + } + + @Test void testUnlexableMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {\n" + + " <¢>\n" + + "};\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 3:3: invalid character '¢'\n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 3:0: mismatched input ' ' expecting EOF\n", + generated.getErrorMessage()); + } + + @Test void testInvalidActionTemplateGroup(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) := <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(209): error compiling action templates file " + actionTemplates + ": Java.stg 1:11: mismatched input ':' expecting '::='\n" + + "error(212): " + grammarFile + ":2:14: error rendering action template: 2:16: no such template: /writeln\n", + generated.getErrorMessage()); + } + + @Test void testInvalidActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ { skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 2:29: premature EOF\n", + generated.getErrorMessage()); + } + + @Test + void testInvalidMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {\n" + + " skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 4:1: premature EOF\n", + generated.getErrorMessage()); + } + + @Test + void testValidActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + @Test + void testValidMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {\n" + + " \n" + + "};\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + @Test + void testActionTemplateHeader(@TempDir Path tempDir) { + String actionTemplates = + "normalizerImports() ::= <<\n" + + "import java.text.Normalizer;\n" + + "import java.text.Normalizer.Form;\n" + + ">>\n" + + "normalize(s) ::= <, Form.NFKC)>>\n" + + "getText() ::= <>\n" + + "setText(s) ::= <);>>"; + + writeActionTemplatesFile(tempDir, actionTemplates); + + String actionTemplatesFile = tempDir + FileSeparator + "Java.stg"; + + String grammar = + "lexer grammar L;\n" + + "@lexer::header {\n" + + "\n" + + "}\n" + + "ID : (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { } ;\n" + + "ID_START : [\\p{XID_Start}] ;\n" + + "ID_CONTINUE: [\\p{XID_Continue}] ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "This _is \ufb01ne", tempDir, actionTemplatesFile); + + String expecting = + "[@0,0:3='This',<1>,1:0]\n" + + "[@1,5:7='_is',<1>,1:5]\n" + + "[@2,9:11='fine',<1>,1:9]\n" + + "[@3,12:11='',<-1>,1:12]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + @Test + void testActionTemplateSemanticPredicate(@TempDir Path tempDir) { + String actionTemplates = "pred() ::= <>"; + + writeActionTemplatesFile(tempDir, actionTemplates); + + String actionTemplatesFile = tempDir + FileSeparator + "Java.stg"; + + String grammar = + "grammar P;\n" + + "file : atom EOF ;\n" + + "atom : scientific | { }? variable ;\n" + + "variable: VARIABLE ;\n" + + "scientific: SCIENTIFIC_NUMBER ;\n" + + "VARIABLE : VALID_ID_START VALID_ID_CHAR* ;\n" + + "SCIENTIFIC_NUMBER : NUMBER (E SIGN? UNSIGNED_INTEGER)? ;\n" + + "fragment VALID_ID_START : ('a' .. 'z') | ('A' .. 'Z') | '_' ;\n" + + "fragment VALID_ID_CHAR : VALID_ID_START | ('0' .. '9') ;\n" + + "fragment NUMBER : ('0' .. '9') + ('.' ('0' .. '9') +)? ;\n" + + "fragment UNSIGNED_INTEGER : ('0' .. '9')+ ;\n" + + "fragment E : 'E' | 'e' ;\n" + + "fragment SIGN : ('+' | '-') ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execParser(grammar, "Bla", tempDir, "file", actionTemplatesFile); + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + ExecutedState executedState = (ExecutedState) state; + + // We can never match the input unless pred() expands to true + assertEquals("", executedState.output); + assertEquals("", executedState.errors); + } + + @Test + void testValidActionTemplateInLibDir(@TempDir Path tempDir, @TempDir Path libDir) { + writeActionTemplatesFile(libDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = "Java.stg"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, libDir, actionTemplates); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + void writeActionTemplatesFile(Path tempDir, String template) { + writeFile(tempDir.toString(), "Java.stg", template); + } + + State execParser(String grammarStr, String input, Path tempDir, String startRule, String actionTemplates) { + RunOptions runOptions = createOptionsForJavaToolTests("P.g4", grammarStr, "PParser", "PLexer", + false, true, startRule, input, + false, false, Stage.Execute, null, actionTemplates); + try (JavaRunner runner = new JavaRunner(tempDir, false)) { + return runner.run(runOptions); + } + } + + State execLexer(String grammarStr, String input, Path tempDir, String actionTemplates) { + RunOptions runOptions = createOptionsForJavaToolTests("L.g4", grammarStr, null, "L", + false, true, null, input, + false, false, Stage.Execute, null, actionTemplates); + try (JavaRunner runner = new JavaRunner(tempDir, false)) { + return runner.run(runOptions); + } + } + + State execLexer(String grammarStr, String input, Path tempDir, Path libDir, String actionTemplates) { + RunOptions runOptions = createOptionsForJavaToolTests("L.g4", grammarStr, null, "L", + false, true, null, input, + false, false, Stage.Execute, libDir.toString(), actionTemplates); + try (JavaRunner runner = new JavaRunner(tempDir, false)) { + return runner.run(runOptions); + } + } +} diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java index d8e8c8c233..b9787d300d 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java @@ -727,7 +727,7 @@ private static boolean compile(String grammarFileName, String grammarStr, String ) { RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammarStr, parserName, null, false, false, startRuleName, null, - false, false, Stage.Compile); + false, false, Stage.Compile, null, null); try (JavaRunner runner = new JavaRunner(tempDirPath, false)) { JavaCompiledState compiledState = (JavaCompiledState) runner.run(runOptions); return !compiledState.containsErrors(); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java index 53f7587381..680fdbf438 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java @@ -392,7 +392,7 @@ private static ParseTreeMatch checkPatternMatch(String grammar, String startRule String lexerName = grammarName+"Lexer"; RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammar, parserName, lexerName, false, false, startRule, input, - false, false, Stage.Execute); + false, false, Stage.Execute, null, null); try (JavaRunner runner = new JavaRunner()) { JavaExecutedState executedState = (JavaExecutedState)runner.run(runOptions); JavaCompiledState compiledState = (JavaCompiledState)executedState.previousState; @@ -413,7 +413,7 @@ private static ParseTreePatternMatcher getPatternMatcher( ) throws Exception { RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammar, parserName, lexerName, false, false, startRule, null, - false, false, Stage.Compile); + false, false, Stage.Compile, null, null); try (JavaRunner runner = new JavaRunner()) { JavaCompiledState compiledState = (JavaCompiledState) runner.run(runOptions); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java index f7db910f79..5f7bc4c82b 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java @@ -223,7 +223,7 @@ public class TestParserProfiler { RunOptions runOptions = createOptionsForJavaToolTests("T.g4", grammar, "TParser", "TLexer", false, false, "s", "xyz;abc;z.q", - true, false, Stage.Execute); + true, false, Stage.Execute, null, null); try (JavaRunner runner = new JavaRunner()) { ExecutedState state = (ExecutedState) runner.run(runOptions); String expecting = diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java index 1a9292a93d..38d0c79d1e 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java @@ -1103,7 +1103,7 @@ protected JavaCompiledState compileJavaParser(boolean leftRecursive) throws IOEx RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, body, parserName, lexerName, false, true, null, null, - false, false, Stage.Compile); + false, false, Stage.Compile, null, null); try (RuntimeRunner runner = new JavaRunner()) { return (JavaCompiledState) runner.run(runOptions); } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java index 572cc3819c..1dda380583 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java @@ -201,7 +201,7 @@ private Pair> compileAndExtract(String grammarFi ) throws Exception { RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammar, parserName, lexerName, false, false, startRuleName, input, - false, false, Stage.Execute); + false, false, Stage.Execute, null, null); try (JavaRunner runner = new JavaRunner()) { JavaExecutedState executedState = (JavaExecutedState)runner.run(runOptions); JavaCompiledState compiledState = (JavaCompiledState)executedState.previousState; diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/ToolTestUtils.java b/tool-testsuite/test/org/antlr/v4/test/tool/ToolTestUtils.java index 6add050bd4..1066a56324 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/ToolTestUtils.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/ToolTestUtils.java @@ -69,7 +69,7 @@ private static ExecutedState execRecognizer(String grammarFileName, String gramm Path workingDir, boolean saveTestDir) { RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammarStr, parserName, lexerName, false, true, startRuleName, input, - false, showDiagnosticErrors, Stage.Execute); + false, showDiagnosticErrors, Stage.Execute, null, null); try (JavaRunner runner = new JavaRunner(workingDir, saveTestDir)) { State result = runner.run(runOptions); if (!(result instanceof ExecutedState)) { @@ -83,11 +83,11 @@ public static RunOptions createOptionsForJavaToolTests( String grammarFileName, String grammarStr, String parserName, String lexerName, boolean useListener, boolean useVisitor, String startRuleName, String input, boolean profile, boolean showDiagnosticErrors, - Stage endStage + Stage endStage, String libDir, String actionTemplates ) { return new RunOptions(grammarFileName, grammarStr, parserName, lexerName, useListener, useVisitor, startRuleName, input, profile, showDiagnosticErrors, false, false, endStage, "Java", - JavaRunner.runtimeTestParserName, PredictionMode.LL, true); + JavaRunner.runtimeTestParserName, libDir, actionTemplates, PredictionMode.LL, true); } public static void testErrors(String[] pairs, boolean printTree) { diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index fb75d82a7a..a423fb33a4 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -1235,6 +1235,48 @@ public enum ErrorType { */ @Deprecated V3_SYNPRED(205, "(...)=> syntactic predicates are not supported in ANTLR 4", ErrorSeverity.ERROR), + /** + * Compiler Error 206. + * + *

cannot find action templates file filename given for grammar

+ */ + CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE(206, "cannot find action templates file given for ", ErrorSeverity.ERROR), + /** + * Compiler Error 207. + * + *

cannot find action templates file filename

+ */ + CANNOT_FIND_ACTION_TEMPLATES_FILE_REFD_IN_GRAMMAR(207, "cannot find action templates file ", ErrorSeverity.ERROR), + /** + * Compiler Error 208. + * + *

error reading action templates file filename: reason

+ */ + ERROR_READING_ACTION_TEMPLATES_FILE(208, "error reading action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 209. + * + *

error compiling action templates file filename: reason

+ */ + ERROR_COMPILING_ACTION_TEMPLATES_FILE(209, "error compiling action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 210. + * + *

error rendering action templates file filename: reason

+ */ + ERROR_RENDERING_ACTION_TEMPLATES_FILE(210, "error rendering action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 211. + * + *

error compiling action template: reason

+ */ + ERROR_COMPILING_ACTION_TEMPLATE(211, "error compiling action template: ", ErrorSeverity.ERROR), + /** + * Compiler Error 212. + * + *

error rendering action template: reason

+ */ + ERROR_RENDERING_ACTION_TEMPLATE(212, "error rendering action template: ", ErrorSeverity.ERROR), // Dependency sorting errors diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index eabee9175f..f49420966d 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -80,6 +80,7 @@ public class Grammar implements AttributeResolver { parserOptions.add("TokenLabelType"); parserOptions.add("tokenVocab"); parserOptions.add("language"); + parserOptions.add("actionTemplates"); parserOptions.add("accessLevel"); parserOptions.add("exportMacro"); parserOptions.add(caseInsensitiveOptionName); @@ -1178,6 +1179,10 @@ public String getLanguage() { return getOptionString("language"); } + public String getActionTemplates() { + return getOptionString("actionTemplates"); + } + public String getOptionString(String key) { return ast.getOptionString(key); } /** Given ^(TOKEN_REF ^(OPTIONS ^(ELEMENT_OPTIONS (= assoc right)))) diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index 5dc00896de..25f5a3ce76 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -7,6 +7,7 @@ package org.antlr.v4.tool; import org.antlr.runtime.CommonToken; +import org.antlr.runtime.RecognitionException; import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.CommonTreeNodeStream; import org.antlr.runtime.tree.Tree; @@ -27,7 +28,13 @@ import org.antlr.v4.tool.ast.GrammarRootAST; import org.antlr.v4.tool.ast.RuleAST; import org.antlr.v4.tool.ast.TerminalAST; +import org.stringtemplate.v4.*; +import org.stringtemplate.v4.compiler.STException; +import org.stringtemplate.v4.misc.*; +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -54,6 +61,10 @@ public void process() { reduceBlocksToSets(root); expandParameterizedLoops(root); + if (g.getActionTemplates() != null) { + expandActionTemplates(root); + } + tool.log("grammar", "after: "+root.toStringTree()); } @@ -509,4 +520,332 @@ public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) { return lexerAST; } + /** + * Create an error listener for .stg action template group files provided via grammar options or via the command line. + */ + private STErrorListener createActionTemplateErrorListener(GrammarAST ast, STGroupFile actionTemplateGroupFile) { + return new STErrorListener() { + private final ErrorManager errorManager = ast.g.tool.errMgr; + + @Override + public void compileTimeError(STMessage stMessage) { + errorManager.toolError( + ErrorType.ERROR_COMPILING_ACTION_TEMPLATES_FILE, + actionTemplateGroupFile.fileName, + stMessage.toString()); + } + + @Override + public void runTimeError(STMessage stMessage) { + errorManager.toolError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATES_FILE, + actionTemplateGroupFile.fileName, + stMessage.toString()); + } + + @Override + public void IOError(STMessage stMessage) { + reportInternalError(stMessage); + } + + @Override + public void internalError(STMessage stMessage) { + reportInternalError(stMessage); + } + + private void reportInternalError(STMessage stMessage) { + errorManager.toolError(ErrorType.INTERNAL_ERROR, stMessage.cause, stMessage.toString()); + } + }; + } + + /** + * Create an error listener for action templates embedded inside a grammar's actions and semantic predicates. + */ + private STErrorListener createGrammarErrorListener(GrammarAST ast) { + return new STErrorListener() { + private final ErrorManager errorManager = ast.g.tool.errMgr; + + /** + * Get the STCompiletimeMesage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTCompiletimeErrorMessage(STCompiletimeMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + if (stMsg.token != null) { + int linePos = ast.getLine() + stMsg.token.getLine() - 1; + int charPos = stMsg.token.getCharPositionInLine(); + if (stMsg.token.getLine() == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + } + + String msg = String.format(stMsg.error.message, stMsg.arg, stMsg.arg2); + + pw.print(msg); + + return sw.toString(); + } + + /** + * Get the STLexerMessage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTLexerErrorMessage(STLexerMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + // From STLexerMessage.toString + RecognitionException re = (RecognitionException)stMsg.cause; + int linePos = ast.getLine() + re.line - 1; + int charPos = re.charPositionInLine; + if (re.line == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + + String msg = String.format(stMsg.error.message, stMsg.msg); + + pw.print(msg); + + return sw.toString(); + } + + @Override + public void compileTimeError(STMessage stMessage) { + if (stMessage instanceof STCompiletimeMessage) { + STCompiletimeMessage compileMessage = (STCompiletimeMessage) stMessage; + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), getSTCompiletimeErrorMessage(compileMessage)); + } + else if (stMessage instanceof STLexerMessage) { + STLexerMessage lexerMessage = (STLexerMessage) stMessage; + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), getSTLexerErrorMessage(lexerMessage)); + } + else { + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), stMessage.toString()); + } + } + + /** + * Get the STRuntimeMessage error location Coordinate. + */ + private Coordinate getSTRuntimeMessageSourceLocation(STRuntimeMessage msg) { + // From STRuntimeMessage.getSourceLocation + if (msg.ip >= 0 && msg.self != null && msg.self.impl != null) { + Interval I = msg.self.impl.sourceMap[msg.ip]; + if (I == null) { + return null; + } else { + int i = I.a; + return Misc.getLineCharPosition(msg.self.impl.template, i); + } + } else { + return null; + } + } + + /** + * Get the STRuntimeMessage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTRuntimeErrorMessage(STRuntimeMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + Coordinate coord = getSTRuntimeMessageSourceLocation(stMsg); + + if (coord != null) { + int linePos = ast.getLine() + coord.line - 1; + int charPos = coord.charPosition; + if (coord.line == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + } + + // From STMessage.toString + String msg = String.format(stMsg.error.message, stMsg.arg, stMsg.arg2, stMsg.arg3); + + pw.print(msg); + + if (stMsg.cause != null) { + pw.print("\nCaused by: "); + stMsg.cause.printStackTrace(pw); + } + + return sw.toString(); + } + + @Override + public void runTimeError(STMessage stMessage) { + if (stMessage instanceof STRuntimeMessage) { + STRuntimeMessage runtimeMessage = (STRuntimeMessage) stMessage; + errorManager.grammarError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE, + ast.g.fileName, + ast.getToken(), + getSTRuntimeErrorMessage(runtimeMessage)); + } else { + errorManager.grammarError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE, + ast.g.fileName, + ast.getToken(), + stMessage.toString()); + } + } + + @Override + public void IOError(STMessage stMessage) { + reportError(stMessage); + } + + @Override + public void internalError(STMessage stMessage) { + reportError(stMessage); + } + + private void reportError(STMessage stMessage) { + errorManager.toolError(ErrorType.INTERNAL_ERROR, stMessage.cause, stMessage.toString()); + } + }; + } + + public File getActionTemplatesGroupFile(GrammarRootAST root, String actionTemplates) { + // Try for an absolute path + File importedFile = new File(actionTemplates); + + if (!importedFile.exists()) { + // Next try the input directory + importedFile = new File(root.g.tool.inputDirectory, actionTemplates); + if (!importedFile.exists()) { + // Next try the parent directory of the grammar file + File grammarFile = new File(root.g.fileName); + String parentDir = grammarFile.getParent(); + importedFile = new File(parentDir, actionTemplates); + if (!importedFile.exists()) { + // Next try the lib directory + importedFile = new File(root.g.tool.libDirectory, actionTemplates); + if (!importedFile.exists()) { + return null; + } + } + } + } + + return importedFile; + } + + public void cannotFindActionTemplatesFileError(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + String actionTemplatesFile = grammar.getActionTemplates(); + + // Check whether this action template file came from options in the grammar file + GrammarAST optionAST = root.getOptionAST("actionTemplates"); + + if (optionAST != null && actionTemplatesFile.equals(optionAST.getToken().getText())) { + errorManager.grammarError( + ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_REFD_IN_GRAMMAR, + grammar.fileName, + optionAST.getToken(), actionTemplatesFile); + } else { + errorManager.toolError( + ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE, + actionTemplatesFile, + grammar.name); + } + } + + public STGroupFile loadActionTemplatesGroupFile(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + String actionTemplatesFile = grammar.getActionTemplates(); + File actionTemplatesGroupFile = getActionTemplatesGroupFile(root, actionTemplatesFile); + + if (actionTemplatesGroupFile == null) { + cannotFindActionTemplatesFileError(root); + return null; + } + + try { + STGroupFile actionTemplates = new STGroupFile(actionTemplatesGroupFile.getAbsolutePath()); + STErrorListener errorListener = createActionTemplateErrorListener(root, actionTemplates); + + // Force load the action templates group file + actionTemplates.setListener(errorListener); + actionTemplates.load(); + + return actionTemplates; + + } catch (IllegalArgumentException e) { + if (e.getMessage() != null && e.getMessage().startsWith("No such group file")) { + cannotFindActionTemplatesFileError(root); + } else { + errorManager.toolError( + ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, + actionTemplatesFile, + e.getMessage()); + } + } catch (STException e) { + errorManager.toolError( + ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, + actionTemplatesFile, + e.getMessage()); + } + + return null; + } + + public void expandActionTemplates(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + + STGroupFile actionTemplates = loadActionTemplatesGroupFile(root); + + if (actionTemplates != null) { + TreeVisitor visitor = new TreeVisitor(new GrammarASTAdaptor()); + visitor.visit(root, new TreeVisitorAction() { + @Override + public Object pre(Object t) { + GrammarAST grammarAST = (GrammarAST) t; + int tokenType = grammarAST.getType(); + if (tokenType == ANTLRParser.ACTION || tokenType == ANTLRParser.SEMPRED) { + return expandActionTemplate((GrammarAST) t, errorManager, actionTemplates); + } + return t; + } + @Override + public Object post(Object t) { + return t; + } + }); + } + } + + public GrammarAST expandActionTemplate(GrammarAST ast, ErrorManager errMgr, STGroupFile actionTemplateGroupFile) { + // Trim the curly braces and trailing question mark + // from an action or semantic predicate + String actionText = ast.getText() + .substring(1, + ast.getType() == ANTLRParser.SEMPRED + ? ast.getText().length() - 2 + : ast.getText().length() - 1); + + STGroupString actionTemplateGroup = new STGroupString( + ast.g.fileName, "action() ::= << " + actionText + " >>"); + + actionTemplateGroup.importTemplates(actionTemplateGroupFile); + + actionTemplateGroup.setListener(createGrammarErrorListener(ast)); + + ST actionTemplate = actionTemplateGroup.getInstanceOf("action"); + + if (actionTemplate != null) { + ast.setText(actionTemplate.render()); + } + + return ast; + } }