Use signed ints for ATN serialization not uint16, except for java #3591

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

parrt merged 46 commits into antlr:dev from parrt:ATN-to-signed-ints

Mar 26, 2022

.circleci/scripts/run-tests-cpp.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,5 +3,6 @@ @@
     set -euo pipefail
     pushd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=4 -Dtest=cpp.** test
     popd

.circleci/scripts/run-tests-dart.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,5 +6,6 @@ dart --version @@
     pushd runtime-testsuite
       echo "running maven tests..."
+      export MAVEN_OPTS="-Xmx8g"
       mvn -Dparallel=classes -DthreadCount=4 -Dtest=dart.** test
     popd

.circleci/scripts/run-tests-dotnet.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,5 +3,6 @@ @@
     set -euo pipefail
     pushd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=4 -Dtest=csharp.** test
     popd

.circleci/scripts/run-tests-go.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,5 +6,6 @@ go version @@
     pushd runtime-testsuite
         echo "running maven tests..."
+        export MAVEN_OPTS="-Xmx8g"
         mvn -Dparallel=classes -DthreadCount=4 -Dtest=go.** test
     popd

.circleci/scripts/run-tests-javascript.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -15,6 +15,7 @@ popd @@
     pushd runtime-testsuite
       echo "running maven tests..."
+      export MAVEN_OPTS="-Xmx8g"
       mvn -Dtest=javascript.** test
       RESULT+=$?
@@ Expand Down @@

.circleci/scripts/run-tests-php.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -7,5 +7,6 @@ php -v @@
     php_path=$(which php)
     pushd runtime-testsuite
       echo "running maven tests..."
+      export MAVEN_OPTS="-Xmx8g"
       mvn -DPHP_PATH="${php_path}" -Dparallel=classes -DthreadCount=4 -Dtest=php.** test
     popd

.circleci/scripts/run-tests-python2.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -17,5 +17,6 @@ python2 --version @@
     pushd runtime-testsuite
       echo "running maven tests..."
+      export MAVEN_OPTS="-Xmx8g"
       mvn -Dparallel=classes -DthreadCount=4 -Dtest=python2.** test
     popd

.circleci/scripts/run-tests-python3.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -17,5 +17,6 @@ python3 --version @@
     pushd runtime-testsuite
       echo "running maven tests..."
+      export MAVEN_OPTS="-Xmx8g"
       mvn -Dparallel=classes -DthreadCount=4 -Dtest=python3.** test
     popd

.circleci/scripts/run-tests-swift.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -17,5 +17,6 @@ set -euo pipefail @@
     pushd runtime-testsuite
       echo "running maven tests..."
+      export MAVEN_OPTS="-Xmx8g"
       mvn -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
     popd

.github/scripts-macosx/run-tests-cpp.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,5 +3,6 @@ @@
     set -euo pipefail
     pushd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=4 -Dtest=cpp.** test
     popd

.github/scripts-macosx/run-tests-dotnet.sh

-Original file line number
+Diff line change
@@ Expand Up @@
     # run tests
     pushd runtime-testsuite/
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=4 -Dtest=csharp.** test
     popd

.github/scripts-macosx/run-tests-swift.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -38,6 +38,7 @@ swift build --version @@
     cd runtime-testsuite/
     #  mvn -e -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
     # I don't know swift enough to make it parallel. revert to single threaded
+    export MAVEN_OPTS="-Xmx8g"
     mvn -e -Dtest=swift.** test
     rc=$?
     cat target/surefire-reports/*.dumpstream || true
@@ Expand Down @@

.github/scripts-windows/run-tests-csharp.cmd

-Original file line number
+Diff line change
@@ -1,5 +1,6 @@
     dotnet build runtime/CSharp/src/Antlr4.csproj -c Release
     dotnet pack runtime/CSharp/src/Antlr4.csproj -c Release
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dtest=csharp.** test
     cd ..

.github/scripts-windows/run-tests-dart.cmd

-Original file line number
+Diff line change
@@ -1,5 +1,6 @@
     C:\ProgramData\chocolatey\bin\choco.exe -y install dart-sdk
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dtest=dart.** test -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat"
     cd ..

.github/scripts-windows/run-tests-go.cmd

-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dtest=go.** test
     cd ..

.github/scripts-windows/run-tests-java.cmd

-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dtest=java.** test
     cd ..

.github/scripts-windows/run-tests-javascript.cmd

-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dtest=javascript.** test
     cd ..

.github/scripts-windows/run-tests-php.cmd

-Original file line number
+Diff line change
@@ Expand Up @@
     move antlr-php-runtime runtime\PHP
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dtest=php.** test -Dantlr-php-php="C:\tools\php81\php.exe"
     cd ..

.github/scripts-windows/run-tests-python2.cmd

-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dantlr-python2-python="C:\Python27\python.exe" -Dtest=python2.** test
     cd ..

.github/scripts-windows/run-tests-python3.cmd

-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     cd runtime-testsuite
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=2 -Dantlr-python3-python="C:\Python310\python.exe" -Dtest=python3.** test
     cd ..

runtime-testsuite/pom.xml

-Original file line number
+Diff line change
@@ Expand Up / @@ -132,6 +132,8 @@ @@
                     <artifactId>maven-compiler-plugin</artifactId>
                     <configuration>
                         <release>8</release>
+                        <source>9</source>
+                        <target>9</target>
                     </configuration>
                 </plugin>
             </plugins>
@@ Expand Down @@

...e-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/TokenType0xFFFF.txt

-Original file line number
+Diff line change
@@ -0,0 +1,13 @@
+    [type]
+    Lexer
+    [grammar]
+    lexer grammar L;
+    T_FFFF: 'FFFF' -> type(65535);
+    [input]
+    FFFF
+    [output]
+    [@0,0:3='FFFF',<65535>,1:0]
+    [@1,4:3='<EOF>',<-1>,1:4]

...me-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/UnicodeCharSet.txt

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,7 @@ Lexer @@
     [grammar]
     lexer grammar L;
-    ID : ([A-Z_]|'Ā'..'') ([A-Z_0-9]|'Ā'..'')*;
+    ID : ([A-Z_]|'Ā'..'\uFFFC') ([A-Z_0-9]|'Ā'..'\uFFFC')*; // FFFD+ are not valid char
     [input]
     均
@@ Expand Down @@

runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -366,9 +366,10 @@ public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(String group, St
  
    		}

    		if (group.equals("LexerExec")) {

    			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorLfTest(targetName));

    			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorCrLfTest(targetName));

    			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorLfDescriptor(targetName));

    			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorCrLfDescriptor(targetName));

    			descriptors.add(GeneratedLexerDescriptors.getLargeLexerDescriptor(targetName));

    			descriptors.add(GeneratedLexerDescriptors.getAtnStatesSizeMoreThan65535Descriptor(targetName));

    		}

    		return descriptors.toArray(new RuntimeTestDescriptor[0]);

runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,6 +6,7 @@ @@
     import org.antlr.v4.runtime.atn.ATN;
     import org.antlr.v4.runtime.atn.ATNDeserializer;
     import org.antlr.v4.runtime.atn.ATNSerializer;
+    import org.antlr.v4.runtime.misc.IntegerList;
     import org.antlr.v4.semantics.SemanticPipeline;
     import org.antlr.v4.tool.Grammar;
     import org.antlr.v4.tool.LexerGrammar;
@@ Expand Down Expand Up / @@ -219,8 +220,9 @@ protected ATN createATN(Grammar g, boolean useSerializer) { @@
     		ATN atn = g.atn;
     		if ( useSerializer ) {
-    			char[] serialized = ATNSerializer.getSerializedAsChars(atn, g.getLanguage());
-    			return new ATNDeserializer().deserialize(serialized);
+    			// sets some flags in ATN
+    			IntegerList serialized = ATNSerializer.getSerialized(atn);
+    			return new ATNDeserializer().deserialize(serialized.toArray());
     		}
     		return atn;
@@ Expand Down @@

runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,7 +1,9 @@
  
    package org.antlr.v4.test.runtime;

    import java.util.*;

    public class GeneratedLexerDescriptors {

    	static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) {

    	static RuntimeTestDescriptor getLineSeparatorLfDescriptor(String targetName) {

    		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();

    		result.name = "LineSeparatorLf";

    		result.targetName = targetName;

    @@ -20,7 +22,7 @@ static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) {
  
    		return result;

    	}

    	static RuntimeTestDescriptor getLineSeparatorCrLfTest(String targetName) {

    	static RuntimeTestDescriptor getLineSeparatorCrLfDescriptor(String targetName) {

    		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();

    		result.name = "LineSeparatorCrLf";

    		result.targetName = targetName;

    @@ -65,4 +67,64 @@ static RuntimeTestDescriptor getLargeLexerDescriptor(String targetName) {
  
    				"[@1,5:4='<EOF>',<-1>,1:5]\n";

    		return result;

    	}

    	static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor(String targetName) {

    		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();

    		result.name = "AtnStatesSizeMoreThan65535";

    		result.notes = "Regression for https://github.com/antlr/antlr4/issues/1863";

    		result.targetName = targetName;

    		result.testType = "Lexer";

    		// I tried playing around with different sizes and I think 1002 works for Go but 1003 does not;

    		// the executing lexer gets a token syntax error for T208 or something like that

    		final int tokensCount = 1024;

    		final String suffix = String.join("", Collections.nCopies(70, "_"));

    		String grammarName = "L";

    		StringBuilder grammar = new StringBuilder();

    		grammar.append("lexer grammar ").append(grammarName).append(";\n");

    		grammar.append('\n');

    		StringBuilder input = new StringBuilder();

    		StringBuilder output = new StringBuilder();

    		int startOffset;

    		int stopOffset = -2;

    		for (int i = 0; i < tokensCount; i++) {

    			String ruleName = String.format("T_%06d", i);

    			String value = ruleName+suffix;

    			grammar.append(ruleName).append(": '").append(value).append("';\n");

    			input.append(value).append('\n');

    			startOffset = stopOffset + 2;

    			stopOffset += value.length() + 1;

    			output.append("[@").append(i).append(',').append(startOffset).append(':').append(stopOffset)

    					.append("='").append(value).append("',<").append(i + 1).append(">,").append(i + 1)

    					.append(":0]\n");

    		}

    		grammar.append("\n");

    		grammar.append("WS: [ \\t\\r\\n]+ -> skip;\n");

    		startOffset = stopOffset + 2;

    		stopOffset = startOffset - 1;

    		output.append("[@").append(tokensCount).append(',').append(startOffset).append(':').append(stopOffset)

    				.append("='<EOF>',<-1>,").append(tokensCount + 1).append(":0]\n");

    		result.grammar = grammar.toString();

    		result.grammarName = grammarName;

    		result.input = input.toString();

    		result.output = output.toString();

    		// We seem to get memory errors and so I am turning this off during CI

    		List<String> all = Arrays.asList(

    //				"CSharp", "Python2", "Python3", "Cpp", "Go", "PHP", "Swift", "Java", "JavaScript", "Node", "Dart"

    				"CSharp", "Python2", "Python3", "Go", "PHP", "Swift", "JavaScript", "Node", "Dart"

    		);

    		result.skipTargets.addAll(all);

    //		result.skipTargets.add("Node"); // doesn't terminate

    //		result.skipTargets.add("PHP"); // "Allowed memory size of 134217728 bytes exhausted (tried to allocate 16384 bytes)..."

    //		result.skipTargets.add("Go"); // syntax error

    		return result;

    	}

    }

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestInterpreterDataReader.java

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -6,20 +6,18 @@
  
    import org.antlr.v4.runtime.atn.ATN;

    import org.antlr.v4.runtime.atn.ATNDeserializer;

    import org.antlr.v4.runtime.atn.ATNSerializer;

    import org.antlr.v4.runtime.misc.IntegerList;

    import org.antlr.v4.runtime.misc.InterpreterDataReader;

    import org.antlr.v4.runtime.misc.Utils;

    import org.antlr.v4.tool.Grammar;

    import org.junit.Assert;

    import org.junit.Test;

    import java.io.IOException;

    import java.lang.reflect.Field;

    import java.net.URL;

    import java.nio.charset.StandardCharsets;

    import java.nio.file.Files;

    import java.nio.file.Path;

    import java.util.ArrayList;

    import java.util.Collection;

    import java.util.List;

    /** This file represents a simple sanity checks on the parsing of the .interp file

    @@ -76,8 +74,8 @@ public void testParseFile() throws IOException, NoSuchFieldException, IllegalAcc
  
    		Assert.assertNull(channels);

    		Assert.assertNull(modes);

            char[] atnChars = ATNSerializer.getSerializedAsChars(atn, g.getLanguage());

    		Assert.assertEquals(ATNDeserializer.SERIALIZED_VERSION, atnChars[0]);

    		IntegerList serialized = ATNSerializer.getSerialized(atn);

    		Assert.assertEquals(ATNDeserializer.SERIALIZED_VERSION, serialized.get(0));

        }

        private <T> List<T> castList(Object obj, Class<T> clazz) {

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/perf/TimeLexerSpeed.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -55,7 +55,7 @@ @@
      *
      *  Sample output on OS X with 4 GHz Intel Core i7 (us == microseconds, 1/1000 of a millisecond):
      *
-     Java VM args: -Xms2G -Xmx2G
+     Java VM args: -Xms2G -Xmx8g
      Warming up Java compiler....
      load_legacy_java_ascii_file average time    53us size  58384b over 3500 loads of 29038 symbols from Parser.java
      load_legacy_java_ascii_file average time    27us size  15568b over 3500 loads of  7625 symbols from RuleContext.java
@@ Expand Down @@

runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -244,7 +244,7 @@ private String locateTool(String tool) { @@
     			return phpPath;
     		}
-    		String[] roots = {"/usr/local/bin/", "/opt/local/bin", "/usr/bin/"};
+    		String[] roots = {"/usr/local/bin/", "/opt/local/bin", "/opt/homebrew/bin/", "/usr/bin/"};
     		for (String root: roots) {
     			if (new File(root + tool).exists()) {
@@ Expand Down @@

runtime/CSharp/src/Atn/ATNDeserializer.cs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -45,8 +45,7 @@ public virtual ATN Deserialize(int[] data)
  
    			ReadRules (atn);

    			ReadModes (atn);

    			IList<IntervalSet> sets = new List<IntervalSet>();

    			ReadSets (atn, sets, ReadInt);

    	        ReadSets (atn, sets, ReadInt32);

    			ReadSets (atn, sets);

    	        ReadEdges (atn, sets);

    			ReadDecisions (atn);

    			ReadLexerActions (atn);

    @@ -190,15 +189,7 @@ protected internal virtual void ReadLexerActions(ATN atn)
  
    				{

    					LexerActionType actionType = (LexerActionType)ReadInt();

    					int data1 = ReadInt();

    					if (data1 == unchecked((int)(0xFFFF)))

    					{

    						data1 = -1;

    					}

    					int data2 = ReadInt();

    					if (data2 == unchecked((int)(0xFFFF)))

    					{

    						data2 = -1;

    					}

    					ILexerAction lexerAction = LexerActionFactory(actionType, data1, data2);

    					atn.lexerActions[i_10] = lexerAction;

    				}

    @@ -309,7 +300,7 @@ protected internal virtual void ReadEdges(ATN atn, IList<IntervalSet> sets)
  
    			}

    		}

    		protected internal virtual void ReadSets(ATN atn, IList<IntervalSet> sets, System.Func<int> readUnicode)

    		protected internal virtual void ReadSets(ATN atn, IList<IntervalSet> sets)

    		{

    			//

    			// SETS

    @@ -327,7 +318,7 @@ protected internal virtual void ReadSets(ATN atn, IList<IntervalSet> sets, Syste
  
    				}

    				for (int j = 0; j < nintervals; j++)

    				{

    					set.Add(readUnicode(), readUnicode());

    					set.Add(ReadInt(), ReadInt());

    				}

    			}

    		}

    @@ -369,9 +360,6 @@ protected internal virtual void ReadRules(ATN atn)
  
    				atn.ruleToStartState[i_5] = startState;

    				if (atn.grammarType == ATNType.Lexer) {

    					int tokenType = ReadInt ();

    					if (tokenType == unchecked((int)(0xFFFF))) {

    						tokenType = TokenConstants.EOF;

    					}

    					atn.ruleToTokenType [i_5] = tokenType;

    				}

    			}

    @@ -967,11 +955,6 @@ protected internal int ReadInt()
  
    			return data[p++];

            }

            protected internal int ReadInt32()

            {

    			return (int)data[p++] | ((int)data[p++] << 16);

            }

            [return: NotNull]

            protected internal virtual Transition EdgeFactory(ATN atn, TransitionType type, int src, int trg, int arg1, int arg2, int arg3, IList<IntervalSet> sets)

            {

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Use signed ints for ATN serialization not uint16, except for java #3591

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!