Switch to using Acorn-based parsing via Acornima (#1820)

* Fix invalid template literal-related bug
* Update test262 suite exclusions

Directory.Build.props

@@ -27,17 +27,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Using Include="Esprima" />
-    <Using Include="Esprima.Ast" />
-    <Using Include="Esprima.Utils" />
-
-    <Using Include="Esprima.Ast.BindingPattern" Alias="DestructuringPattern" />
-    <Using Include="Esprima.Ast.Module" Alias="AstModule" />
-    <Using Include="Esprima.Ast.Nodes" Alias="NodeType" />
-    <Using Include="Esprima.JavaScriptParser" Alias="Parser" />
-    <Using Include="Esprima.Location" Alias="SourceLocation" />
-    <Using Include="Esprima.TokenType" Alias="TokenKind" />
-    <Using Include="Esprima.ParserException" Alias="ParseErrorException" />
+    <Using Include="Acornima" />
+    <Using Include="Acornima.Ast" />
+    <Using Include="Acornima.Ast.Module" Alias="AstModule" />
   </ItemGroup>
 
 </Project>

Directory.Packages.props

@@ -4,9 +4,10 @@
     <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="Acornima" Version="1.0.0" />
+    <PackageVersion Include="Acornima.Extras" Version="1.0.0" />
     <PackageVersion Include="BenchmarkDotNet" Version="0.13.12" />
     <PackageVersion Include="BenchmarkDotNet.TestAdapter" Version="0.13.12" />
-    <PackageVersion Include="Esprima" Version="3.0.5" />
     <PackageVersion Include="FluentAssertions" Version="6.12.0" />
     <PackageVersion Include="Flurl.Http.Signed" Version="3.2.4" />
     <PackageVersion Include="Jurassic" Version="3.2.7" />

Jint.Benchmark/Jint.Benchmark.csproj

@@ -32,8 +32,7 @@
     <PackageReference Include="YantraJS.Core" />
   </ItemGroup>
   <ItemGroup>
-    <Using Include="Esprima" />
-    <Using Include="Esprima.Ast" />
-    <Using Include="Esprima.JavaScriptParser" Alias="Parser" />
+    <Using Include="Acornima" />
+    <Using Include="Acornima.Ast" />
   </ItemGroup>
 </Project>

Jint.Repl/Program.cs

@@ -43,7 +43,6 @@
 var parsingOptions = new ScriptParsingOptions
 {
     Tolerant = true,
-    CompileRegex = false,
 };
 
 var serializer = new JsonSerializer(engine);

Jint.Tests.CommonScripts/Jint.Tests.CommonScripts.csproj

@@ -17,6 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Acornima.Extras" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="NUnit" />
     <PackageReference Include="NUnit3TestAdapter" />

Jint.Tests.PublicInterface/Jint.Tests.PublicInterface.csproj

@@ -22,6 +22,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Acornima.Extras" />
     <PackageReference Include="Flurl.Http.Signed" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="MongoDB.Bson.signed" />

Jint.Tests.PublicInterface/RavenApiUsageTests.cs

@@ -18,16 +18,15 @@ public void CanBuildCustomScriptFunctionInstance()
 
         var properties = new List<Node>
         {
-            new Property(PropertyKind.Init, new Identifier("field"), false,
-                new StaticMemberExpression(new Identifier("self"), new Identifier("field"), optional: false), false, false)
+            new ObjectProperty(PropertyKind.Init, new Identifier("field"),
+                new MemberExpression(new Identifier("self"), new Identifier("field"), computed: false, optional: false), false, false, false)
         };
 
         var functionExp = new FunctionExpression(
             new Identifier("functionId"),
             NodeList.Create<Node>(new List<Expression> { new Identifier("self") }),
-            new BlockStatement(NodeList.Create(new List<Statement> { new ReturnStatement(new ObjectExpression(NodeList.Create(properties))) })),
+            new FunctionBody(NodeList.Create(new List<Statement> { new ReturnStatement(new ObjectExpression(NodeList.Create(properties))) }), strict: false),
             generator: false,
-            strict: false,
             async: false);
 
         var functionObject = new ScriptFunction(

Jint.Tests.Test262/Jint.Tests.Test262.csproj

@@ -16,6 +16,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Acornima.Extras" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="NUnit" />
     <PackageReference Include="NUnit3TestAdapter" />

Jint.Tests.Test262/Test262Harness.settings.json

@@ -8,9 +8,9 @@
     "Array.fromAsync",
     "async-iteration",
     "Atomics",
+    "decorators",
     "import-assertions",
     "iterator-helpers",
-    "regexp-duplicate-named-groups",
     "regexp-lookbehind",
     "regexp-unicode-property-escapes",
     "regexp-v-flag",
@@ -43,6 +43,24 @@
     "language/literals/regexp/u-case-mapping.js",
     "built-ins/RegExp/lookahead-quantifier-match-groups.js",
     "built-ins/RegExp/unicode_full_case_folding.js",
+    "built-ins/RegExp/named-groups/duplicate-names-exec.js",
+    "built-ins/RegExp/named-groups/duplicate-names-exec.js",
+    "built-ins/RegExp/named-groups/duplicate-names-match.js",
+    "built-ins/RegExp/named-groups/duplicate-names-match.js",
+    "built-ins/RegExp/named-groups/duplicate-names-match-indices.js",
+    "built-ins/RegExp/named-groups/duplicate-names-match-indices.js",
+    "built-ins/RegExp/named-groups/duplicate-names-replace.js",
+    "built-ins/RegExp/named-groups/duplicate-names-replace.js",
+    "built-ins/RegExp/named-groups/duplicate-names-replaceall.js",
+    "built-ins/RegExp/named-groups/duplicate-names-replaceall.js",
+    "built-ins/RegExp/prototype/exec/duplicate-named-groups-properties.js",
+    "built-ins/RegExp/prototype/exec/duplicate-named-groups-properties.js",
+    "built-ins/RegExp/prototype/exec/duplicate-named-indices-groups-properties.js",
+    "built-ins/RegExp/prototype/exec/duplicate-named-indices-groups-properties.js",
+    "built-ins/String/prototype/match/duplicate-named-groups-properties.js",
+    "built-ins/String/prototype/match/duplicate-named-groups-properties.js",
+    "built-ins/String/prototype/match/duplicate-named-indices-groups-properties.js",
+    "built-ins/String/prototype/match/duplicate-named-indices-groups-properties.js",
 
     // requires investigation how to process complex function name evaluation for property
     "built-ins/Function/prototype/toString/method-computed-property-name.js",
@@ -68,10 +86,6 @@
     // C# can't distinguish 1.797693134862315808e+308 and 1.797693134862315708145274237317e+308
     "language/types/number/8.5.1.js",
 
-    // inner binding is immutable (from parameters) Expected SameValue(«null», «function() {{ ... }}») to be true
-    "language/expressions/function/scope-name-var-open-non-strict.js",
-    "language/expressions/function/scope-name-var-open-strict.js",
-
     // generators
     "built-ins/GeneratorFunction/instance-yield-expr-in-param.js",
     "built-ins/GeneratorFunction/instance-yield-expr-in-param.js",
@@ -1018,23 +1032,13 @@
     "language/statements/variable/dstr/ary-ptrn-elision.js",
     "language/statements/variable/dstr/ary-ptrn-elision.js",
 
-     // Esprima problem
-    "language/expressions/object/let-non-strict-access.js",
-    "language/expressions/object/let-non-strict-syntax.js",
-    "language/expressions/object/yield-non-strict-access.js",
-    "language/expressions/object/yield-non-strict-syntax.js",
-    "language/expressions/tagged-template/invalid-escape-sequences.js",
-    "language/statements/for-of/dstr-obj-id-identifier-yield-ident-valid.js",
-    "language/statements/for/head-lhs-let.js",
-
     // SharedArrayBuffer not implemented
     "built-ins/SharedArrayBuffer/prototype/prop-desc.js",
 
     // special casing data
     "built-ins/**/special_casing*.js",
 
     // failing tests in new test suite (due to updating to latest and using whole set)
-    "language/arguments-object/mapped/nonconfigurable-descriptors-define-failure.js",
     "language/eval-code/direct/arrow-fn-a-following-parameter-is-named-arguments-arrow-func-declare-arguments-assign-incl-def-param-arrow-arguments.js",
     "language/eval-code/direct/arrow-fn-a-following-parameter-is-named-arguments-arrow-func-declare-arguments-assign.js",
     "language/eval-code/direct/arrow-fn-a-preceding-parameter-is-named-arguments-arrow-func-declare-arguments-assign-incl-def-param-arrow-arguments.js",
@@ -1203,7 +1207,6 @@
     "language/expressions/assignment/dstr/array-elem-put-obj-literal-prop-ref-init.js",
     "language/expressions/assignment/dstr/obj-prop-elem-target-obj-literal-prop-ref-init-active.js",
     "language/expressions/assignment/dstr/obj-prop-elem-target-obj-literal-prop-ref-init.js",
-    "language/expressions/assignment/fn-name-lhs-member.js",
     "language/expressions/assignment/target-member-computed-reference-null.js",
     "language/expressions/assignment/target-member-computed-reference-undefined.js",
     "language/expressions/assignment/target-member-identifier-reference-null.js",
@@ -1243,8 +1246,6 @@
     "language/statements/for-of/dstr/obj-id-init-let.js",
     "language/statements/for-of/dstr/obj-prop-elem-target-obj-literal-prop-ref-init-active.js",
     "language/statements/for-of/dstr/obj-prop-elem-target-obj-literal-prop-ref-init.js",
-    "language/statements/for-of/head-lhs-async-dot.js",
-    "language/statements/for-of/head-lhs-async-escaped.js",
     "language/statements/function/cptn-decl.js",
     "language/statements/function/dstr/ary-init-iter-get-err-array-prototype.js",
     "language/statements/function/dstr/dflt-ary-init-iter-get-err-array-prototype.js",

Jint.Tests.Test262/Test262Test.cs

@@ -39,7 +39,7 @@ private Engine BuildTestExecutor(Test262File file)
 
                 var script = Engine.PrepareScript(args.At(0).AsString(), options: new ScriptPreparationOptions
                 {
-                    ParsingOptions = ScriptParsingOptions.Default with { CompileRegex = false, Tolerant = false },
+                    ParsingOptions = ScriptParsingOptions.Default with { Tolerant = false },
                 });
 
                 return engine.Evaluate(script);
@@ -96,7 +96,7 @@ private static void ExecuteTest(Engine engine, Test262File file)
         {
             var script = Engine.PrepareScript(file.Program, source: file.FileName, options: new ScriptPreparationOptions
             {
-                ParsingOptions = ScriptParsingOptions.Default with { CompileRegex = false, Tolerant = false },
+                ParsingOptions = ScriptParsingOptions.Default with { Tolerant = false },
             });
 
             engine.Execute(script);

Jint.Tests/Jint.Tests.csproj

@@ -24,6 +24,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Acornima.Extras" />
     <PackageReference Include="FluentAssertions" />
     <PackageReference Include="Flurl.Http.Signed" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" />

Jint.Tests/Parser/JavascriptParserTests.cs

@@ -52,10 +52,10 @@ public void ShouldParseBinaryExpression()
         Assert.Single(body);
         Assert.NotNull(binary = body.First().As<ExpressionStatement>().Expression.As<BinaryExpression>());
         Assert.Equal(3d, binary.Right.As<Literal>().Value);
-        Assert.Equal(BinaryOperator.Times, binary.Operator);
+        Assert.Equal(Operator.Multiplication, binary.Operator);
         Assert.Equal(1d, binary.Left.As<BinaryExpression>().Left.As<Literal>().Value);
         Assert.Equal(2d, binary.Left.As<BinaryExpression>().Right.As<Literal>().Value);
-        Assert.Equal(BinaryOperator.Plus, binary.Left.As<BinaryExpression>().Operator);
+        Assert.Equal(Operator.Addition, binary.Left.As<BinaryExpression>().Operator);
     }
 
     [Theory]
@@ -154,7 +154,7 @@ public void ShouldProvideLocationForMultiLinesStringLiterals()
 \
 '
 ";
-        var program = new Parser(new ParserOptions()).ParseScript(Code);
+        var program = new Parser().ParseScript(Code);
         var expr = program.Body.First().As<ExpressionStatement>().Expression;
         Assert.Equal(1, expr.Location.Start.Line);
         Assert.Equal(0, expr.Location.Start.Column);
@@ -165,7 +165,7 @@ public void ShouldProvideLocationForMultiLinesStringLiterals()
     [Fact]
     public void ShouldThrowErrorForInvalidLeftHandOperation()
     {
-        Assert.Throws<JavaScriptException>(() => new Engine().Execute("~ (WE0=1)--- l('1');"));
+        Assert.Throws<SyntaxErrorException>(() => new Engine().Execute("~ (WE0=1)--- l('1');"));
     }
 
 
@@ -176,6 +176,6 @@ public void ShouldThrowErrorForInvalidLeftHandOperation()
     [InlineData("-.-")]
     public void ShouldThrowParseErrorExceptionForInvalidCode(string code)
     {
-        Assert.Throws<ParseErrorException>(() => new Parser().ParseScript(code));
+        Assert.Throws<SyntaxErrorException>(() => new Parser().ParseScript(code));
     }
 }

Jint.Tests/Runtime/Debugger/BreakPointTests.cs

@@ -157,7 +157,7 @@ public void BreakPointBreaksInCorrectSource()
             bool didBreak = false;
             engine.Debugger.Break += (sender, info) =>
             {
-                Assert.Equal("script2", info.Location.Source);
+                Assert.Equal("script2", info.Location.SourceFile);
                 Assert.Equal(3, info.Location.Start.Line);
                 Assert.Equal(0, info.Location.Start.Column);
                 didBreak = true;

Jint.Tests/Runtime/Debugger/EvaluateTests.cs

@@ -72,7 +72,7 @@ function test(x)
             {
                 var exception = Assert.Throws<DebugEvaluationException>(() =>
                     engine.Debugger.Evaluate("this is a syntax error"));
-                Assert.IsType<ParseErrorException>(exception.InnerException);
+                Assert.IsType<SyntaxErrorException>(exception.InnerException);
             });
         }
 

Jint.Tests/Runtime/Debugger/StepFlowTests.cs

@@ -38,11 +38,11 @@ public void StepsThroughWhileLoop()
             Assert.Collection(nodes,
                 node => Assert.IsType<VariableDeclaration>(node), // let x = 0;
                 node => Assert.IsType<WhileStatement>(node),      // while ...
-                node => Assert.IsType<BinaryExpression>(node),    // x < 2
-                node => Assert.IsType<ExpressionStatement>(node), // x++;
-                node => Assert.IsType<BinaryExpression>(node),    // x < 2
-                node => Assert.IsType<ExpressionStatement>(node), // x++;
-                node => Assert.IsType<BinaryExpression>(node)     // x < 2 (false)
+                node => Assert.IsType<NonLogicalBinaryExpression>(node),    // x < 2
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // x++;
+                node => Assert.IsType<NonLogicalBinaryExpression>(node),    // x < 2
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // x++;
+                node => Assert.IsType<NonLogicalBinaryExpression>(node)     // x < 2 (false)
             );
         }
 
@@ -63,10 +63,10 @@ public void StepsThroughDoWhileLoop()
             Assert.Collection(nodes,
                 node => Assert.IsType<VariableDeclaration>(node), // let x = 0;
                 node => Assert.IsType<DoWhileStatement>(node),    // do ...
-                node => Assert.IsType<ExpressionStatement>(node), // x++;
-                node => Assert.IsType<BinaryExpression>(node),    // x < 2
-                node => Assert.IsType<ExpressionStatement>(node), // x++;
-                node => Assert.IsType<BinaryExpression>(node)     // x < 2 (false)
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // x++;
+                node => Assert.IsType<NonLogicalBinaryExpression>(node),    // x < 2
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // x++;
+                node => Assert.IsType<NonLogicalBinaryExpression>(node)     // x < 2 (false)
             );
         }
 
@@ -85,13 +85,13 @@ public void StepsThroughForLoop()
             Assert.Collection(nodes,
                 node => Assert.IsType<ForStatement>(node),        // for ...
                 node => Assert.IsType<VariableDeclaration>(node), // let x = 0
-                node => Assert.IsType<BinaryExpression>(node),    // x < 2
+                node => Assert.IsType<NonLogicalBinaryExpression>(node),    // x < 2
                 node => Assert.True(node.IsLiteral("dummy")),     // 'dummy';
                 node => Assert.IsType<UpdateExpression>(node),    // x++;
-                node => Assert.IsType<BinaryExpression>(node),    // x < 2
+                node => Assert.IsType<NonLogicalBinaryExpression>(node),    // x < 2
                 node => Assert.True(node.IsLiteral("dummy")),     // 'dummy';
                 node => Assert.IsType<UpdateExpression>(node),    // x++;
-                node => Assert.IsType<BinaryExpression>(node)     // x < 2 (false)
+                node => Assert.IsType<NonLogicalBinaryExpression>(node)     // x < 2 (false)
             );
         }
 
@@ -135,9 +135,9 @@ public void StepsThroughForInLoop()
                 node => Assert.IsType<VariableDeclaration>(node), // let obj = { x: 1, y: 2 };
                 node => Assert.IsType<ForInStatement>(node),      // for ...
                 node => Assert.IsType<VariableDeclaration>(node), // key
-                node => Assert.IsType<ExpressionStatement>(node), // 'dummy';
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // 'dummy';
                 node => Assert.IsType<VariableDeclaration>(node), // key
-                node => Assert.IsType<ExpressionStatement>(node)  // 'dummy';
+                node => Assert.IsType<NonSpecialExpressionStatement>(node)  // 'dummy';
             );
         }
 
@@ -160,7 +160,7 @@ class Test
 
             Assert.Collection(nodes,
                 node => Assert.IsType<ClassDeclaration>(node),          // class Test
-                node => Assert.IsType<ExpressionStatement>(node),       // new Test();
+                node => Assert.IsType<NonSpecialExpressionStatement>(node),       // new Test();
                 node => Assert.True(node.IsLiteral("in constructor")),  // 'in constructor()'
                 node => Assert.Null(node),                              // return point
                 node => Assert.True(node.IsLiteral("after construction"))
@@ -182,7 +182,7 @@ function test()
 
             Assert.Collection(nodes,
                 node => Assert.IsType<FunctionDeclaration>(node), // function(test) ...;
-                node => Assert.IsType<ExpressionStatement>(node), // test();
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // test();
                 node => Assert.True(node.IsLiteral("dummy")),     // 'dummy';
                 node => Assert.Null(node)                         // return point
             );
@@ -202,7 +202,7 @@ class Test
             var nodes = CollectStepNodes(script);
             Assert.Collection(nodes,
                 node => Assert.IsType<ClassDeclaration>(node),    // class Test
-                node => Assert.IsType<ExpressionStatement>(node), // new Test();
+                node => Assert.IsType<NonSpecialExpressionStatement>(node), // new Test();
                 node => Assert.True(node.IsLiteral("dummy"))      // 'dummy';
             );
         }

Jint.Tests/Runtime/Debugger/TestHelpers.cs

@@ -9,7 +9,7 @@ public static bool IsLiteral(this Node node, string requiredValue = null)
             return node switch
             {
                 Directive directive => requiredValue == null || directive.Value == requiredValue,
-                ExpressionStatement expr => requiredValue == null || (expr.Expression is Literal literal && literal.StringValue == requiredValue),
+                NonSpecialExpressionStatement expr => requiredValue == null || (expr.Expression is StringLiteral literal && literal.Value == requiredValue),
                 _ => false
             };
         }