Skip to content

use ints not chars for C# #3567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
parrt merged 5 commits into from
Mar 5, 2022
Merged

use ints not chars for C# #3567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c5f0b2d
use ints not chars for C#
parrt Mar 5, 2022
a1fc8e1
oops. remove 'quotes'
parrt Mar 5, 2022
c8fb3a4
regen from XPathLexer.g4
parrt Mar 5, 2022
46fab6c
simplify ATN with bypass alts mechanism in Java.
parrt Mar 5, 2022
f51f695
Change string to int[] for serialized ATN for C#; removed unneeded `u…
parrt Mar 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions runtime/CSharp/src/Atn/ATNDeserializer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ public ATNDeserializer(ATNDeserializationOptions deserializationOptions)
this.deserializationOptions = deserializationOptions;
}

char[] data;
int[] data;
int p;

public virtual ATN Deserialize(char[] data)
public virtual ATN Deserialize(int[] data)
{
this.data = data;
CheckVersion ();
Expand Down
8 changes: 4 additions & 4 deletions runtime/CSharp/src/Misc/RuleDependencyChecker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,12 +330,12 @@ private static void GetElementDependencies(ICustomAttributeProvider annotatedEle

private static RuleDependencyChecker.RuleRelations ExtractRuleRelations(TypeInfo recognizer)
{
string serializedATN = GetSerializedATN(recognizer);
int[] serializedATN = GetSerializedATN(recognizer);
if (serializedATN == null)
{
return null;
}
ATN atn = new ATNDeserializer().Deserialize(serializedATN.ToCharArray());
ATN atn = new ATNDeserializer().Deserialize(serializedATN);
RuleDependencyChecker.RuleRelations relations = new RuleDependencyChecker.RuleRelations(atn.ruleToStartState.Length);
foreach (ATNState state in atn.states)
{
Expand All @@ -356,11 +356,11 @@ private static RuleDependencyChecker.RuleRelations ExtractRuleRelations(TypeInfo
return relations;
}

private static string GetSerializedATN(TypeInfo recognizerClass)
private static int[] GetSerializedATN(TypeInfo recognizerClass)
{
FieldInfo serializedAtnField = recognizerClass.DeclaredFields.First(i => i.Name == "_serializedATN");
if (serializedAtnField != null)
return (string)serializedAtnField.GetValue(null);
return (int[])serializedAtnField.GetValue(null);

if (recognizerClass.BaseType != null)
return GetSerializedATN(recognizerClass.BaseType.GetTypeInfo());
Expand Down
21 changes: 10 additions & 11 deletions runtime/CSharp/src/Parser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ public virtual void ExitEveryRule(ParserRuleContext ctx)
/// bypass alternatives.
/// </summary>
/// <seealso cref="Antlr4.Runtime.Atn.ATNDeserializationOptions.GenerateRuleBypassTransitions()"/>
private static readonly IDictionary<string, ATN> bypassAltsAtnCache = new Dictionary<string, ATN>();
// private static readonly IDictionary<string, ATN> bypassAltsAtnCache = new Dictionary<string, ATN>();
private ATN bypassAltsAtnCache;

/// <summary>The error handling strategy for the parser.</summary>
/// <remarks>
Expand Down Expand Up @@ -562,22 +563,20 @@ public virtual ITokenFactory TokenFactory
[return: NotNull]
public virtual ATN GetATNWithBypassAlts()
{
string serializedAtn = SerializedAtn;
int[] serializedAtn = SerializedAtn;
if (serializedAtn == null)
{
throw new NotSupportedException("The current parser does not support an ATN with bypass alternatives.");
}
lock (bypassAltsAtnCache)
lock (this)
{
ATN result = bypassAltsAtnCache.Get(serializedAtn);
if (result == null)
{
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
deserializationOptions.GenerateRuleBypassTransitions = true;
result = new ATNDeserializer(deserializationOptions).Deserialize(serializedAtn.ToCharArray());
bypassAltsAtnCache.Put(serializedAtn, result);
if ( bypassAltsAtnCache!=null ) {
return bypassAltsAtnCache;
}
return result;
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
deserializationOptions.GenerateRuleBypassTransitions = true;
bypassAltsAtnCache = new ATNDeserializer(deserializationOptions).Deserialize(serializedAtn);
return bypassAltsAtnCache;
}
}

Expand Down
2 changes: 1 addition & 1 deletion runtime/CSharp/src/Recognizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public virtual int GetTokenType(string tokenName)
/// <p>For interpreters, we don't know their serialized ATN despite having
/// created the interpreter from it.</p>
/// </remarks>
public virtual string SerializedAtn
public virtual int[] SerializedAtn
{
[return: NotNull]
get
Expand Down
6 changes: 5 additions & 1 deletion runtime/CSharp/src/Tree/Pattern/ParseTreePatternMatcher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,11 @@ public virtual ParseTreePattern Compile(string pattern, int patternRuleIndex)
IList<IToken> tokenList = Tokenize(pattern);
ListTokenSource tokenSrc = new ListTokenSource(tokenList);
CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
ParserInterpreter parserInterp = new ParserInterpreter(parser.GrammarFileName, parser.Vocabulary, Arrays.AsList(parser.RuleNames), parser.GetATNWithBypassAlts(), tokens);
ParserInterpreter parserInterp = new ParserInterpreter(parser.GrammarFileName,
parser.Vocabulary,
Arrays.AsList(parser.RuleNames),
parser.GetATNWithBypassAlts(),
tokens);
IParseTree tree = null;
try
{
Expand Down
78 changes: 24 additions & 54 deletions runtime/CSharp/src/Tree/Xpath/XPathLexer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
// Ambiguous reference in cref attribute
#pragma warning disable 419


using System;

using System;
using System.IO;
using System.Text;
Expand All @@ -46,7 +43,7 @@ public const int
};

public static readonly string[] ruleNames = {
"Anywhere", "Root", "Wildcard", "Bang", "ID", "NameChar", "NameStartChar",
"Anywhere", "Root", "Wildcard", "Bang", "ID", "NameChar", "NameStartChar",
"String"
};

Expand All @@ -64,7 +61,7 @@ public XPathLexer(ICharStream input, TextWriter output, TextWriter errorOutput)
null, null, null, "'//'", "'/'", "'*'", "'!'"
};
private static readonly string[] _SymbolicNames = {
null, "TokenRef", "RuleRef", "Anywhere", "Root", "Wildcard", "Bang", "ID",
null, "TokenRef", "RuleRef", "Anywhere", "Root", "Wildcard", "Bang", "ID",
Copy link
Member

@KvanTTT KvanTTT Mar 6, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Excess whitespace.

"String"
};
public static readonly IVocabulary DefaultVocabulary = new Vocabulary(_LiteralNames, _SymbolicNames);
Expand All @@ -86,7 +83,7 @@ public override IVocabulary Vocabulary

public override string[] ModeNames { get { return modeNames; } }

public override string SerializedAtn { get { return new string(_serializedATN); } }
public override int[] SerializedAtn { get { return _serializedATN; } }

static XPathLexer() {
decisionToDFA = new DFA[_ATN.NumberOfDecisions];
Expand All @@ -101,61 +98,34 @@ public override void Action(RuleContext _localctx, int ruleIndex, int actionInde
}
private void ID_action(RuleContext _localctx, int actionIndex) {
switch (actionIndex) {
case 0:
case 0:
String text = Text;
if ( Char.IsUpper(text[0]) )
if ( Char.IsUpper(text[0]) )
Type = TokenRef;
else
else
Type = RuleRef;
break;
}
}

private static char[] _serializedATN = {
'\x4', '\0', '\b', '\x32', '\x6', '\xFFFF', '\x2', '\0', '\a', '\0', '\x2',
'\x1', '\a', '\x1', '\x2', '\x2', '\a', '\x2', '\x2', '\x3', '\a', '\x3',
'\x2', '\x4', '\a', '\x4', '\x2', '\x5', '\a', '\x5', '\x2', '\x6', '\a',
'\x6', '\x2', '\a', '\a', '\a', '\x1', '\0', '\x1', '\0', '\x1', '\0',
'\x1', '\x1', '\x1', '\x1', '\x1', '\x2', '\x1', '\x2', '\x1', '\x3',
'\x1', '\x3', '\x1', '\x4', '\x1', '\x4', '\x5', '\x4', '\x1D', '\b',
'\x4', '\n', '\x4', '\f', '\x4', ' ', '\t', '\x4', '\x1', '\x4', '\x1',
'\x4', '\x1', '\x5', '\x1', '\x5', '\x3', '\x5', '&', '\b', '\x5', '\x1',
'\x6', '\x1', '\x6', '\x1', '\a', '\x1', '\a', '\x5', '\a', ',', '\b',
'\a', '\n', '\a', '\f', '\a', '/', '\t', '\a', '\x1', '\a', '\x1', '\a',
'\x1', '-', '\0', '\b', '\x1', '\x3', '\x3', '\x4', '\x5', '\x5', '\a',
'\x6', '\t', '\a', '\v', '\0', '\r', '\0', '\xF', '\b', '\x1', '\0', '\x2',
'\x5', '\0', '\x30', '\x39', '_', '_', '\xB7', '\xB7', '\x300', '\x36F',
'\x203F', '\x2040', '\r', '\0', '\x41', 'Z', '\x61', 'z', '\xC0', '\xD6',
'\xD8', '\xF6', '\xF8', '\x2FF', '\x370', '\x37D', '\x37F', '\x1FFF',
'\x200C', '\x200D', '\x2070', '\x218F', '\x2C00', '\x2FEF', '\x3001',
'\xD7FF', '\xF900', '\xFDCF', '\xFDF0', '\xFFFD', '\0', '\x32', '\0',
'\x1', '\x1', '\0', '\0', '\0', '\0', '\x3', '\x1', '\0', '\0', '\0',
'\0', '\x5', '\x1', '\0', '\0', '\0', '\0', '\a', '\x1', '\0', '\0', '\0',
'\0', '\t', '\x1', '\0', '\0', '\0', '\0', '\xF', '\x1', '\0', '\0', '\0',
'\x1', '\x11', '\x1', '\0', '\0', '\0', '\x3', '\x14', '\x1', '\0', '\0',
'\0', '\x5', '\x16', '\x1', '\0', '\0', '\0', '\a', '\x18', '\x1', '\0',
'\0', '\0', '\t', '\x1A', '\x1', '\0', '\0', '\0', '\v', '%', '\x1', '\0',
'\0', '\0', '\r', '\'', '\x1', '\0', '\0', '\0', '\xF', ')', '\x1', '\0',
'\0', '\0', '\x11', '\x12', '\x5', '/', '\0', '\0', '\x12', '\x13', '\x5',
'/', '\0', '\0', '\x13', '\x2', '\x1', '\0', '\0', '\0', '\x14', '\x15',
'\x5', '/', '\0', '\0', '\x15', '\x4', '\x1', '\0', '\0', '\0', '\x16',
'\x17', '\x5', '*', '\0', '\0', '\x17', '\x6', '\x1', '\0', '\0', '\0',
'\x18', '\x19', '\x5', '!', '\0', '\0', '\x19', '\b', '\x1', '\0', '\0',
'\0', '\x1A', '\x1E', '\x3', '\r', '\x6', '\0', '\x1B', '\x1D', '\x3',
'\v', '\x5', '\0', '\x1C', '\x1B', '\x1', '\0', '\0', '\0', '\x1D', ' ',
'\x1', '\0', '\0', '\0', '\x1E', '\x1C', '\x1', '\0', '\0', '\0', '\x1E',
'\x1F', '\x1', '\0', '\0', '\0', '\x1F', '!', '\x1', '\0', '\0', '\0',
' ', '\x1E', '\x1', '\0', '\0', '\0', '!', '\"', '\x6', '\x4', '\0', '\0',
'\"', '\n', '\x1', '\0', '\0', '\0', '#', '&', '\x3', '\r', '\x6', '\0',
'$', '&', '\a', '\0', '\0', '\0', '%', '#', '\x1', '\0', '\0', '\0', '%',
'$', '\x1', '\0', '\0', '\0', '&', '\f', '\x1', '\0', '\0', '\0', '\'',
'(', '\a', '\x1', '\0', '\0', '(', '\xE', '\x1', '\0', '\0', '\0', ')',
'-', '\x5', '\'', '\0', '\0', '*', ',', '\t', '\0', '\0', '\0', '+', '*',
'\x1', '\0', '\0', '\0', ',', '/', '\x1', '\0', '\0', '\0', '-', '.',
'\x1', '\0', '\0', '\0', '-', '+', '\x1', '\0', '\0', '\0', '.', '\x30',
'\x1', '\0', '\0', '\0', '/', '-', '\x1', '\0', '\0', '\0', '\x30', '\x31',
'\x5', '\'', '\0', '\0', '\x31', '\x10', '\x1', '\0', '\0', '\0', '\x4',
'\0', '\x1E', '%', '-', '\x1', '\x1', '\x4', '\0',
private static int[] _serializedATN = {
4,0,8,50,6,65535,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,
6,2,7,7,7,1,0,1,0,1,0,1,1,1,1,1,2,1,2,1,3,1,3,1,4,1,4,5,4,29,8,4,10,4,
12,4,32,9,4,1,4,1,4,1,5,1,5,3,5,38,8,5,1,6,1,6,1,7,1,7,5,7,44,8,7,10,7,
12,7,47,9,7,1,7,1,7,1,45,0,8,1,3,3,4,5,5,7,6,9,7,11,0,13,0,15,8,1,0,2,
5,0,48,57,95,95,183,183,768,879,8255,8256,13,0,65,90,97,122,192,214,216,
246,248,767,880,893,895,8191,8204,8205,8304,8591,11264,12271,12289,55295,
63744,64975,65008,65533,0,50,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,
0,0,0,0,9,1,0,0,0,0,15,1,0,0,0,1,17,1,0,0,0,3,20,1,0,0,0,5,22,1,0,0,0,
7,24,1,0,0,0,9,26,1,0,0,0,11,37,1,0,0,0,13,39,1,0,0,0,15,41,1,0,0,0,17,
18,5,47,0,0,18,19,5,47,0,0,19,2,1,0,0,0,20,21,5,47,0,0,21,4,1,0,0,0,22,
23,5,42,0,0,23,6,1,0,0,0,24,25,5,33,0,0,25,8,1,0,0,0,26,30,3,13,6,0,27,
29,3,11,5,0,28,27,1,0,0,0,29,32,1,0,0,0,30,28,1,0,0,0,30,31,1,0,0,0,31,
33,1,0,0,0,32,30,1,0,0,0,33,34,6,4,0,0,34,10,1,0,0,0,35,38,3,13,6,0,36,
38,7,0,0,0,37,35,1,0,0,0,37,36,1,0,0,0,38,12,1,0,0,0,39,40,7,1,0,0,40,
14,1,0,0,0,41,45,5,39,0,0,42,44,9,0,0,0,43,42,1,0,0,0,44,47,1,0,0,0,45,
46,1,0,0,0,45,43,1,0,0,0,46,48,1,0,0,0,47,45,1,0,0,0,48,49,5,39,0,0,49,
16,1,0,0,0,4,0,30,37,45,1,1,4,0
};

public static readonly ATN _ATN =
Expand Down
4 changes: 0 additions & 4 deletions runtime/CSharp/src/Tree/Xpath/XPathLexer.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
lexer grammar XPathLexer;

@header {
using System;
}

tokens { TokenRef, RuleRef }

/*
Expand Down
25 changes: 12 additions & 13 deletions runtime/Java/src/org/antlr/v4/runtime/Parser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,14 @@ public void exitEveryRule(ParserRuleContext ctx) {
}

/**
* This field maps from the serialized ATN string to the deserialized {@link ATN} with
* bypass alternatives.
* This field holds the deserialized {@link ATN} with bypass alternatives, created
* lazily upon first demand. In 4.10 I changed from map<serializedATNstring, ATN>
* since we only need one per parser object and also it complicates other targets
* that don't use ATN strings.
*
* @see ATNDeserializationOptions#isGenerateRuleBypassTransitions()
*/
private static final Map<String, ATN> bypassAltsAtnCache =
new WeakHashMap<String, ATN>();
private ATN bypassAltsAtnCache;

/**
* The error handling strategy for the parser. The default value is a new
Expand Down Expand Up @@ -445,16 +446,14 @@ public ATN getATNWithBypassAlts() {
throw new UnsupportedOperationException("The current parser does not support an ATN with bypass alternatives.");
}

synchronized (bypassAltsAtnCache) {
ATN result = bypassAltsAtnCache.get(serializedAtn);
if (result == null) {
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
deserializationOptions.setGenerateRuleBypassTransitions(true);
result = new ATNDeserializer(deserializationOptions).deserialize(serializedAtn.toCharArray());
bypassAltsAtnCache.put(serializedAtn, result);
synchronized (this) {
if ( bypassAltsAtnCache!=null ) {
return bypassAltsAtnCache;
}

return result;
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
deserializationOptions.setGenerateRuleBypassTransitions(true);
bypassAltsAtnCache = new ATNDeserializer(deserializationOptions).deserialize(serializedAtn.toCharArray());
return bypassAltsAtnCache;
}
}

Expand Down
9 changes: 4 additions & 5 deletions tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ public partial class <parser.name> : <superClass; null="Parser"> {

public override string[] RuleNames { get { return ruleNames; } }

public override string SerializedAtn { get { return new string(_serializedATN); } }
public override int[] SerializedAtn { get { return _serializedATN; } }

static <parser.name>() {
decisionToDFA = new DFA[_ATN.NumberOfDecisions];
Expand Down Expand Up @@ -1033,7 +1033,7 @@ public partial class <lexer.name> : <superClass; null="Lexer"> {

public override string[] ModeNames { get { return modeNames; } }

public override string SerializedAtn { get { return new string(_serializedATN); } }
public override int[] SerializedAtn { get { return _serializedATN; } }

static <lexer.name>() {
decisionToDFA = new DFA[_ATN.NumberOfDecisions];
Expand All @@ -1046,10 +1046,9 @@ public partial class <lexer.name> : <superClass; null="Lexer"> {
}
>>


SerializedATN(model) ::= <<
private static char[] _serializedATN = {
<model.serialized: {s | '<s>'}; separator=",", wrap>
private static int[] _serializedATN = {
<model.serialized: {s | <s>}; separator=",", wrap>
};

public static readonly ATN _ATN =
Expand Down
2 changes: 1 addition & 1 deletion tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ private void reportError(STMessage msg) {

@Override
public boolean isATNSerializedAsInts() {
return false;
return true;
}

@Override
Expand Down