Optimize ATN serialization data

runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java

@@ -396,12 +396,10 @@ private void generateParser(String grammarFileName,
 		List<String> files = new ArrayList<>();
 		if (lexerName != null) {
 			files.add(lexerName + ".swift");
-			files.add(lexerName + "ATN.swift");
 		}
 
 		if (parserName != null) {
 			files.add(parserName + ".swift");
-			files.add(parserName + "ATN.swift");
 			Set<String> optionsSet = new HashSet<>(Arrays.asList(extraOptions));
 			String grammarName = grammarFileName.substring(0, grammarFileName.lastIndexOf('.'));
 			if (!optionsSet.contains("-no-listener")) {

runtime/Swift/Sources/Antlr4/Parser.swift

@@ -423,7 +423,7 @@ open class Parser: Recognizer<ParserATNSimulator> {
 
             var opts = ATNDeserializationOptions()
             opts.generateRuleBypassTransitions = true
-            let result = try! ATNDeserializer(opts).deserialize(Array(serializedAtn))
+            let result = try! ATNDeserializer(opts).deserialize(serializedAtn)
             bypassAltsAtnCache[serializedAtn] = result
             return result
         }

runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift

@@ -22,10 +22,10 @@ public class ATNDeserializer {
         self.deserializationOptions = deserializationOptions ?? ATNDeserializationOptions()
     }
 
-    public func deserialize(_ inData: [Character]) throws -> ATN {
-        let data = inData.map { Int(integerLiteral: $0.unicodeValue) }
-
+    public func deserialize(_ str: String) throws -> ATN {
+        let data = str.utf16.map { element in Int(element) }
         var p = 0
+
         let version = data[p]
         p += 1
         if version != ATNDeserializer.SERIALIZED_VERSION {
@@ -57,8 +57,7 @@ public class ATNDeserializer {
 
             var ruleIndex = data[p]
             p += 1
-            if ruleIndex == Int.max {
-                // Character.MAX_VALUE
+            if ruleIndex == UInt16.max {
                 ruleIndex = -1
             }
 
@@ -117,7 +116,7 @@ public class ATNDeserializer {
             if atn.grammarType == ATNType.lexer {
                 var tokenType = data[p]
                 p += 1
-                if tokenType == 0xFFFF {
+                if tokenType == UInt16.max {
                     tokenType = CommonToken.EOF
                 }
 
@@ -180,12 +179,14 @@ public class ATNDeserializer {
         //
         let ndecisions = data[p]
         p += 1
-        for i in 1...ndecisions {
-            let s = data[p]
-            p += 1
-            let decState = atn.states[s] as! DecisionState
-            atn.appendDecisionToState(decState)
-            decState.decision = i - 1
+        if (ndecisions >= 1) {
+            for i in 1...ndecisions {
+                let s = data[p]
+                p += 1
+                let decState = atn.states[s] as! DecisionState
+                atn.appendDecisionToState(decState)
+                decState.decision = i - 1
+            }
         }
 
         //
@@ -200,13 +201,13 @@ public class ATNDeserializer {
                 p += 1
                 var data1 = data[p]
                 p += 1
-                if data1 == 0xFFFF {
+                if data1 == UInt16.max {
                     data1 = -1
                 }
 
                 var data2 = data[p]
                 p += 1
-                if data2 == 0xFFFF {
+                if data2 == UInt16.max {
                     data2 = -1
                 }
 
@@ -220,7 +221,6 @@ public class ATNDeserializer {
         return atn
     }
 
-
     private func readUnicodeInt(_ data: [Int], _ p: inout Int) -> Int {
         let result = data[p]
         p += 1
@@ -254,201 +254,6 @@ public class ATNDeserializer {
         }
     }
 
-    public func deserializeFromJson(_ jsonStr: String) -> ATN {
-        guard !jsonStr.isEmpty else {
-            fatalError("ATN Serialization is empty,Please include *LexerATN.json and  *ParserATN.json in TARGETS-Build Phases-Copy Bundle Resources")
-        }
-        if let JSONData = jsonStr.data(using: .utf8) {
-            do {
-                let JSON = try JSONSerialization.jsonObject(with: JSONData, options: JSONSerialization.ReadingOptions(rawValue: 0))
-                guard let JSONDictionary = JSON as? [String: Any] else {
-                    fatalError("deserializeFromJson Not a Dictionary")
-                }
-
-                return try dictToJson(JSONDictionary)
-
-            } catch let JSONError {
-                print("\(JSONError)")
-            }
-        }
-
-        fatalError("Could not deserialize ATN ")
-    }
-
-    public func dictToJson(_ dict: [String: Any]) throws -> ATN {
-        let version = dict["version"] as! Int
-        if version != ATNDeserializer.SERIALIZED_VERSION {
-            let reason = "Could not deserialize ATN with version \(version) (expected \(ATNDeserializer.SERIALIZED_VERSION))."
-            throw ANTLRError.unsupportedOperation(msg: reason)
-        }
-
-        let grammarType = ATNType(rawValue: dict["grammarType"] as! Int)!
-        let maxTokenType = dict["maxTokenType"] as! Int
-        let atn = ATN(grammarType, maxTokenType)
-
-        //
-        // STATES
-        //
-        var loopBackStateNumbers = [(LoopEndState, Int)]()
-        var endStateNumbers = [(BlockStartState, Int)]()
-
-        let states = dict["states"] as! [[String: Any]]
-
-        for state in states {
-            let ruleIndex = state["ruleIndex"] as! Int
-            let stype = state["stateType"] as! Int
-            let s = try stateFactory(stype, ruleIndex)!
-            if stype == ATNState.LOOP_END {
-                // special case
-                let loopBackStateNumber = state["detailStateNumber"] as! Int
-                loopBackStateNumbers.append((s as! LoopEndState, loopBackStateNumber))
-            }
-            else if let bsState = s as? BlockStartState {
-                let endStateNumber = state["detailStateNumber"] as! Int
-                endStateNumbers.append((bsState, endStateNumber))
-            }
-            atn.addState(s)
-        }
-
-        // delay the assignment of loop back and end states until we know all the state instances have been initialized
-        for pair in loopBackStateNumbers {
-            pair.0.loopBackState = atn.states[pair.1]
-        }
-
-        for pair in endStateNumbers {
-            pair.0.endState = atn.states[pair.1] as? BlockEndState
-        }
-
-        let numNonGreedyStates = dict["nonGreedyStates"] as! [Int]
-        for numNonGreedyState in numNonGreedyStates {
-            (atn.states[numNonGreedyState] as! DecisionState).nonGreedy = true
-        }
-
-        let numPrecedenceStates = dict["precedenceStates"] as! [Int]
-        for numPrecedenceState in numPrecedenceStates {
-            (atn.states[numPrecedenceState] as! RuleStartState).isPrecedenceRule = true
-        }
-
-        //
-        // RULES
-        //
-        let ruleToStartState = dict["ruleToStartState"] as! [[String: Any]]
-        let nrules = ruleToStartState.count
-        var ruleToTokenType = [Int]()
-        var ruleToStartStateParsed = [RuleStartState]()
-        for i in 0..<nrules {
-            let currentRuleToStartState = ruleToStartState[i]
-            let s = currentRuleToStartState["stateNumber"] as! Int
-            let startState = atn.states[s] as! RuleStartState
-            ruleToStartStateParsed.append(startState)
-
-            if atn.grammarType == ATNType.lexer {
-                var tokenType = currentRuleToStartState["ruleToTokenType"] as! Int
-                if tokenType == -1 {
-                    tokenType = CommonToken.EOF
-                }
-                ruleToTokenType.append(tokenType)
-            }
-        }
-        atn.ruleToStartState = ruleToStartStateParsed
-        if atn.grammarType == ATNType.lexer {
-            atn.ruleToTokenType = ruleToTokenType
-        }
-
-        fillRuleToStopState(atn)
-
-        //
-        // MODES
-        //
-        let modeToStartState = dict["modeToStartState"] as! [Int]
-        for stateNumber in modeToStartState {
-            atn.appendModeToStartState(atn.states[stateNumber] as! TokensStartState)
-        }
-
-        //
-        // SETS
-        //
-        var sets = [IntervalSet]()
-        let nsets = dict["nsets"] as! Int
-        let intervalSet = dict["IntervalSet"] as! [[String: Any]]
-
-        for i in 0..<nsets {
-            let setBuilder = intervalSet[i]
-            let nintervals = setBuilder["size"] as! Int
-
-            let set = IntervalSet()
-            sets.append(set)
-
-            let containsEof = (setBuilder["containsEof"] as! Int) != 0
-            if containsEof {
-                try! set.add(-1)
-            }
-            let intervalsBuilder = setBuilder["Intervals"] as! [[String: Any]]
-
-            for j in 0..<nintervals {
-                let vals = intervalsBuilder[j]
-                try! set.add((vals["a"] as! Int), (vals["b"] as! Int))
-            }
-        }
-
-
-        //
-        // EDGES
-        //
-        let allTransitions = dict["allTransitionsBuilder"] as! [[[String: Any]]]
-
-        for transitionsBuilder in allTransitions {
-            for transition in transitionsBuilder {
-                let src = transition["src"] as! Int
-                let trg = transition["trg"] as! Int
-                let ttype = transition["edgeType"] as! Int
-                let arg1 = transition["arg1"] as! Int
-                let arg2 = transition["arg2"] as! Int
-                let arg3 = transition["arg3"] as! Int
-                let trans = try edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets)
-
-                let srcState = atn.states[src]!
-                srcState.addTransition(trans)
-            }
-        }
-
-        deriveEdgesForRuleStopStates(atn)
-        try validateStates(atn)
-
-        //
-        // DECISIONS
-        //
-        let ndecisions = dict["decisionToState"] as! [Int]
-        let length = ndecisions.count
-        for i in 0..<length {
-            let s = ndecisions[i]
-            let decState = atn.states[s] as! DecisionState
-            atn.appendDecisionToState(decState)
-            decState.decision = i
-        }
-
-        //
-        // LEXER ACTIONS
-        //
-        if atn.grammarType == ATNType.lexer {
-            let lexerActionsBuilder = dict["lexerActions"] as! [[String: Any]]
-
-            var lexerActions = [LexerAction]()
-            for lexerActionDict in lexerActionsBuilder {
-                let actionTypeValue = lexerActionDict["actionType"] as! Int
-                let actionType = LexerActionType(rawValue: actionTypeValue)!
-                let data1 = lexerActionDict["a"] as! Int
-                let data2 = lexerActionDict["b"] as! Int
-                let lexerAction = lexerActionFactory(actionType, data1, data2)
-                lexerActions.append(lexerAction)
-            }
-            atn.lexerActions = lexerActions
-        }
-
-        try finalizeATN(atn)
-        return atn
-    }
-
     private func fillRuleToStopState(_ atn: ATN) {
         let nrules = atn.ruleToStartState.count
         atn.ruleToStopState = [RuleStopState](repeating: RuleStopState(), count: nrules)

runtime/Swift/Sources/Antlr4/misc/InterpreterDataReader.swift

@@ -109,7 +109,7 @@ public class InterpreterDataReader {
         self.ruleNames = ruleNames
         self.channelNames = channelNames
         self.modeNames = modeNames
-        let atnSerialized = atnText.map{Character(UnicodeScalar(UInt16($0.trimmingCharacters(in:.whitespaces))!)!)}
+        let atnSerialized = String(atnText.map{Character(UnicodeScalar(UInt16($0.trimmingCharacters(in:.whitespaces))!)!)})
         atn = try ATNDeserializer().deserialize(atnSerialized)
     }
 

tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg

@@ -1049,7 +1049,7 @@ public partial class <lexer.name> : <superClass; null="Lexer"> {
 
 SerializedATN(model) ::= <<
 private static char[] _serializedATN = {
-	<model.serialized; separator=", ", wrap>,
+	<model.serialized: {s | '<s>'}; separator=",", wrap>
 };
 
 public static readonly ATN _ATN =

tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg

@@ -383,13 +383,12 @@ static std::vector\<uint16_t> _serializedATN;
 
 // Constructs the serialized ATN and writes init code for static member vars.
 SerializedATN(model) ::= <<
-<model.segments: {segment | static const uint16_t serializedATNSegment<i0>[] = {
-  <segment; wrap={<\n>   }>
-\};}; separator="\n">
+static const uint16_t serializedATNSegment[] = {
+	<model.serialized: {s | <s>}; separator=",", wrap>
+};
 
-<model.segments: {segment | _serializedATN.insert(_serializedATN.end(), serializedATNSegment<i0>,
-  serializedATNSegment<i0> + sizeof(serializedATNSegment<i0>) / sizeof(serializedATNSegment<i0>[0]));
-}>
+_serializedATN.insert(_serializedATN.end(), serializedATNSegment,
+  serializedATNSegment + sizeof(serializedATNSegment) / sizeof(serializedATNSegment[0]));
 
 atn::ATNDeserializer deserializer;
 _atn = deserializer.deserialize(_serializedATN);

tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg

@@ -853,17 +853,8 @@ class <lexer.name> extends <superClass; null="Lexer"> {
 >>
 
 SerializedATN(model) ::= <<
-<if(rest(model.segments))>
-<! requires segmented representation !>
-<model.segments:{segment|static final String _serializedATNSegment<i0> =
-  '<segment; wrap={'<\n><\t>'}>';}; separator="\n">
-static final String _serializedATN = [
-    <model.segments:{segment | _serializedATNSegment<i0>}; separator=",\n">
-  ].join();
-<else>
-<! only one segment, can be inlined !>
 static const String _serializedATN = '<model.serialized; wrap={'<\n><\t>'}>';
-<endif>
+
 static final ATN _ATN =
     ATNDeserializer().deserialize(_serializedATN.codeUnits);
 >>

tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg

@@ -1209,19 +1209,8 @@ namespace<if(lexerFile.genPackage)> <lexerFile.genPackage><endif> {
 >>
 
 SerializedATN(model) ::= <<
-<if(rest(model.segments))>
-/**
- * @var string
- */
-private const SERIALIZED_ATN =
-	<model.segments:{segment| "<segment; wrap={" .<\n>"}>"}; separator=" .\n">;
-<else>
-/**
- * @var string
- */
 private const SERIALIZED_ATN =
 	"<model.serialized; wrap={" .<\n>    "}>";
-<endif>
 >>
 
 /**