Project/aot trimming/building

.github/workflows/dotnetcore.yml

@@ -43,7 +43,7 @@ jobs:
         output: 'lcov.net7.0.info'
         threshold: 80
         outputFormat: 'lcov'
-        excludes: '[program]*,[expressionParser]*,[jsonparser]*,[while]*,[indentedWhile]*,[SimpleExpressionParser]*,[GenericLexerWithCallbacks]*,[indented]*,[postProcessedLexerParser]*,[XML]*,[SimpleTemplate]*'
+        excludes: '[program]*,[expressionParser]*,[jsonparser]*,[while]*,[indentedWhile]*,[SimpleExpressionParser]*,[GenericLexerWithCallbacks]*,[indented]*,[postProcessedLexerParser]*,[XML]*,[SimpleTemplate]*,[GeneratedXML]*,[ExplicitTokens]*,[postProcessedLexerParser]*,[GenericLexerWithCallbacks]*'
 
     - name: coveralls
       uses: coverallsapp/[email protected]

.gitignore

@@ -42,3 +42,7 @@ coverage.json
 ruleset1.ruleset
 opencov.bat
 BenchmarkDotNet.Artifacts
+
+*/*.nupkg
+*/*.snupkg
+src/sly/nupkg/*

sly.sln

@@ -39,6 +39,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XML", "src/samples/XML/XML.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleTemplate", "src/samples/SimpleTemplate/SimpleTemplate.csproj", "{BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GeneratedXML", "src\samples\GeneratedXML\GeneratedXML.csproj", "{7FEF0D32-FFC7-4886-A4B7-71314C25627E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExplicitTokens", "src\samples\ExplicitTokens\ExplicitTokens.csproj", "{94B9B654-82CB-4969-8DCA-250916E53724}"
+EndProject
 Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
@@ -120,6 +124,14 @@ Global
 		{BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7FEF0D32-FFC7-4886-A4B7-71314C25627E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7FEF0D32-FFC7-4886-A4B7-71314C25627E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7FEF0D32-FFC7-4886-A4B7-71314C25627E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7FEF0D32-FFC7-4886-A4B7-71314C25627E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{94B9B654-82CB-4969-8DCA-250916E53724}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{94B9B654-82CB-4969-8DCA-250916E53724}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{94B9B654-82CB-4969-8DCA-250916E53724}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{94B9B654-82CB-4969-8DCA-250916E53724}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -136,6 +148,8 @@ Global
 		{8F703C3C-0D10-4F43-AC2E-14843D59E460} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
 		{81E72CFA-A6D6-4DB4-B3B5-E167064E7858} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
 		{BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
+		{7FEF0D32-FFC7-4886-A4B7-71314C25627E} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
+		{94B9B654-82CB-4969-8DCA-250916E53724} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {43254130-CF3E-480E-952F-E50CA5D2E417}

src/CslyGenerator.Tests/CslyGenerator.Tests.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+
+        <RootNamespace>CslyGenerator.Tests</RootNamespace>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.XUnit" Version="1.1.1"/>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2"/>
+        <PackageReference Include="xunit" Version="2.4.2"/>
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\src\sly\sly.csproj" />
+    </ItemGroup>
+
+
+</Project>

src/CslyGenerator.Tests/SourceGeneratorTests.cs

@@ -0,0 +1,255 @@
+using System.IO;
+using System.Linq;
+using sly.sourceGenerator;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Xunit;
+
+namespace CslyGenerator.Tests;
+
+public class SourceGeneratorTests
+{
+    private const string parserClassTest = $@"
+using aot.lexer;
+using sly.lexer;
+using sly.parser.generator;
+
+namespace TestNamespace;
+
+public enum AotTestLexer
+{{
+    [Double]
+    DOUBLE,
+    [AlphaId]
+    IDENTIFIER,
+    [Sugar(""+"")]
+    PLUS,
+    [Sugar(""++"")]
+    INCREMENT,
+    [Sugar(""-"")]
+    MINUS,
+    [Sugar(""*"")]
+    TIMES,
+    [Sugar(""/"")]
+    DIVIDE,
+    [Sugar(""("")]
+    LPAREN,
+    [Sugar("")"")]
+    RPAREN,
+    [Sugar(""!"")]
+    FACTORIAL,
+
+
+
+    [Push(""carré"")]
+    [Sugar(""²"")]
+    SQUARE,
+[Mode(""carré"")]
+[Lexeme(""$-$"")]
+REGEX,
+
+[Lexeme(GenericToken.KeyWord,""hello"")]
+HELLO,
+[Lexeme(GenericToken.String)]
+STR,
+[Lexeme(GenericToken.Int)]
+I,
+[MultiLineComment(""<!--"",""-->"",channel:Channels.Main)]
+        [Mode]
+        COMMENT,
+}}
+
+[ParserRoot(""root"")]
+[UseMemoization]
+[UseBroadentokenWindow]
+[AutoCloseIndentationsAttribute]
+public class AotTestParser
+{{
+    [Production(""root : SimpleExpressionParser_expressions"")]
+    public double Root(double value) => value;
+
+    [Operation((int) AotTestLexer.PLUS, Affix.InFix, Associativity.Right, 10)]
+    [Operation(""MINUS"", Affix.InFix, Associativity.Left, 10)]
+    public double BinaryTermExpression(double left, Token<AotTestLexer> operation, double right)
+    {{
+        double result = 0;
+        switch (operation.TokenID)
+        {{
+            case AotTestLexer.PLUS:
+            {{
+                result = left + right;
+                break;
+            }}
+            case AotTestLexer.MINUS:
+            {{
+                result = left - right;
+                break;
+            }}
+        }}
+
+        return result;
+    }}
+
+
+    [Operation((int) AotTestLexer.TIMES, Affix.InFix, Associativity.Right, 50)]
+    [Operation(""DIVIDE"", Affix.InFix, Associativity.Left, 50)]
+    [NodeName(""multiplication_or_division"")]
+    public double BinaryFactorExpression(double left, Token<AotTestLexer> operation, double right)
+    {{
+        double result = 0;
+        switch (operation.TokenID)
+        {{
+            case AotTestLexer.TIMES:
+            {{
+                result = left * right;
+                break;
+            }}
+            case AotTestLexer.DIVIDE:
+            {{
+                result = left / right;
+                break;
+            }}
+        }}
+
+        return result;
+    }}
+
+
+    [Prefix((int) AotTestLexer.MINUS,  Associativity.Right, 100)]
+    public double PreFixExpression(Token<AotTestLexer> operation, double value)
+    {{
+        return -value;
+    }}
+
+    [Postfix((int) AotTestLexer.FACTORIAL, Associativity.Right, 100)]
+    public double PostFixExpression(double value, Token<AotTestLexer> operation)
+    {{
+        if (operation.TokenID == AotTestLexer.SQUARE)
+        {{
+            return value * value;
+        }}
+        if(operation.TokenID == AotTestLexer.FACTORIAL || operation.Value == ""!"")
+        {{
+            var factorial = 1;
+            for (var i = 1; i <= value; i++) factorial *= i;
+            return factorial;
+        }}
+        return value;
+    }}
+
+    [Operand]
+    [Production(""operand : primary_value"")]
+    [NodeName(""double"")]
+    public double OperandValue(double value)
+    {{
+        return value;
+    }}
+
+
+    [Production(""primary_value : DOUBLE"")]
+    [NodeName(""double"")]
+    public double OperandDouble(Token<AotTestLexer> value)
+    {{
+        return value.DoubleValue;
+    }}
+
+    [Production(""primary_value : INT"")]
+    [NodeName(""integer"")]
+    public double OperandInt(Token<AotTestLexer> value)
+    {{
+        return value.DoubleValue;
+    }}
+
+    [Production(""primary_value : LPAREN SimpleExpressionParser_expressions RPAREN"")]
+    [NodeName(""group"")]
+    public double OperandGroup(Token<AotTestLexer> lparen, double value, Token<AotTestLexer> rparen)
+    {{
+        return value;
+    }}
+}}
+
+[ParserGenerator(typeof(AotTestLexer), typeof(AotTestParser), typeof(double))]
+public partial class TestGenerator
+{{
+
+}}
+
+ public enum ExtendedLexer 
+    {{
+        [Extension]
+        EXT,
+
+        [Sugar(""-"")]
+        DASH,
+
+        [AlphaNumId]
+        ID
+
+    }}
+
+    public class ExtendedLexerParser {{ }}
+
+    [ParserGenerator(typeof(ExtendedLexer),typeof(ExtendedLexerParser),typeof(string))]
+    public partial class ExtendedLexerGenerator : AbstractParserGenerator<ExtendedLexer>
+    {{
+        public override Action<ExtendedLexer, LexemeAttribute, GenericLexer<ExtendedLexer>> UseTokenExtensions()
+        {{
+            var e = (ExtendedLexer token, LexemeAttribute lexem, GenericLexer<ExtendedLexer> lexer) =>
+            {{
+                if (token == ExtendedLexer.EXT)
+                {{
+                    NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
+                    {{
+                        match.Properties[GenericLexer<ExtendedLexer>.DerivedToken] = ExtendedLexer.EXT;
+                        return match;
+                    }};
+
+                    var fsmBuilder = lexer.FSMBuilder;
+
+
+                    fsmBuilder.GoTo(GenericLexer<ExtendedLexer>.start) 
+                        .Transition('$')
+                        .Transition('_')
+                        .Transition('$')
+                        .End(GenericToken.Extension) // mark as ending node 
+                        .CallBack(callback); // set the ending callback
+                }}
+            }};
+            return e;
+        }}
+    }}
+
+
+
+"; 
+
+    [Fact]
+    public void ParserGeneratorTest()
+    {
+        // Create an instance of the source generator.
+        var generator = new CslyParserGenerator();
+
+        // Source generators should be tested using 'GeneratorDriver'.
+        var driver = CSharpGeneratorDriver.Create(new[] { generator });
+
+        // To run generators, we can use an empty compilation.
+        var compilation = CSharpCompilation.Create(nameof(ParserGeneratorTest),
+            new[] { CSharpSyntaxTree.ParseText(parserClassTest) },
+            new[]
+            {
+                // To support 'System.Attribute' inheritance, add reference to 'System.Private.CoreLib'.
+                MetadataReference.CreateFromFile(typeof(object).Assembly.Location)
+            });
+
+        // Run generators. Don't forget to use the new compilation rather than the previous one.
+        var runResult = driver.RunGenerators(compilation).GetRunResult();
+
+        var generatedFiles = runResult.GeneratedTrees.Select(x => new FileInfo(x.FilePath).Name).ToArray();
+
+        Assert.Equivalent(new[]
+        {
+            "TestGenerator.g.cs",
+            "ExtendedLexerGenerator.g.cs"
+        }, generatedFiles);
+    }
+}

src/CslyGenerator.Tests/Utils/TestAdditionalFile.cs

@@ -0,0 +1,20 @@
+using System.Threading;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Text;
+
+namespace CslyGenerator.Tests.Utils;
+
+public class TestAdditionalFile : AdditionalText
+{
+    private readonly SourceText _text;
+
+    public TestAdditionalFile(string path, string text)
+    {
+        Path = path;
+        _text = SourceText.From(text);
+    }
+
+    public override SourceText GetText(CancellationToken cancellationToken = new()) => _text;
+
+    public override string Path { get; }
+}

src/benchCurrent/GenericLexerBench.cs

@@ -21,7 +21,7 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp70);
+                var baseJob = Job.MediumRun.WithToolchain(CsProjCoreToolchain.NetCoreApp70);
             }
         }
 

src/benchCurrent/JsonParserBench.cs

@@ -23,7 +23,7 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp70);
+                var baseJob = Job.MediumRun.WithToolchain(CsProjCoreToolchain.NetCoreApp70);
             }
         }
 

src/benchCurrent/SimpleExpressionBench.cs

@@ -23,7 +23,7 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp70);
+                var baseJob = Job.MediumRun.WithToolchain(CsProjCoreToolchain.NetCoreApp70);
             }
         }
 

src/benchCurrent/WhileBench.cs

@@ -27,7 +27,7 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp70);
+                var baseJob = Job.MediumRun.WithToolchain(CsProjCoreToolchain.NetCoreApp70);
             }
         }