From 634330c894f9f0700294b98853427339f9447448 Mon Sep 17 00:00:00 2001 From: Rekkonnect <8298332+Rekkonnect@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:35:09 +0300 Subject: [PATCH 01/44] Add failing test --- .../IncrementalParsingTests.cs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs index fa72f15c6145d..136a176c1a4f7 100644 --- a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs @@ -584,6 +584,62 @@ int LocalFunc() WalkTreeAndVerify(tree.GetCompilationUnitRoot(), fullTree.GetCompilationUnitRoot()); } + [Fact] + public void TestCollectionExpressionSpreadVsDeletingTopLevelBrace() + { + var source = """ + namespace Example; + + public sealed class Program + { + public void M2() + { + const bool condition = true; + int[] values = [1, 2, 3]; + if (condition) + { + { + if (condition) + { + values[1] = 312; + } + } + } + if (condition) + { + values = [.. values]; + } + } + } + """; + var tree = SyntaxFactory.ParseSyntaxTree(source); + Assert.Empty(tree.GetDiagnostics()); + + const string valueSetterLine = "values[1] = 312;"; + var text = tree.GetText(); + var valueSetterLinePosition = source.IndexOf(valueSetterLine); + var lines = text.Lines; + + int valueSetterLineIndex = lines.IndexOf(valueSetterLinePosition); + var openBraceLine = text.Lines[valueSetterLineIndex - 1]; + Assert.EndsWith("{", openBraceLine.ToString()); + text = text.WithChanges(new TextChange(openBraceLine.SpanIncludingLineBreak, "")); + tree = tree.WithChangedText(text); + + lines = text.Lines; + var closeBraceLine = text.Lines[valueSetterLineIndex]; + Assert.EndsWith("}", closeBraceLine.ToString()); + text = text.WithChanges(new TextChange(closeBraceLine.SpanIncludingLineBreak, "")); + tree = tree.WithChangedText(text); + + Assert.Empty(tree.GetDiagnostics()); + + var fullTree = SyntaxFactory.ParseSyntaxTree(text.ToString()); + Assert.Empty(fullTree.GetDiagnostics()); + + WalkTreeAndVerify(tree.GetCompilationUnitRoot(), fullTree.GetCompilationUnitRoot()); + } + #region "Regression" #if false From 44645f77caf8ff539f46c724145dbc39bd8be2c5 Mon Sep 17 00:00:00 2001 From: Rekkonnect <8298332+Rekkonnect@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:46:20 +0300 Subject: [PATCH 02/44] WorkItem :) --- .../Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs index 136a176c1a4f7..41408f9a85259 100644 --- a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs @@ -585,6 +585,7 @@ int LocalFunc() } [Fact] + [WorkItem("https://github.com/dotnet/roslyn/issues/74456")] public void TestCollectionExpressionSpreadVsDeletingTopLevelBrace() { var source = """ From d6e55b4f791791e372c1aac7aa75065e62fae80d Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 12:11:05 -0700 Subject: [PATCH 03/44] Renames and docs --- .../IncrementalParsingTests.cs | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs index 41408f9a85259..b225d392486ad 100644 --- a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs @@ -584,11 +584,10 @@ int LocalFunc() WalkTreeAndVerify(tree.GetCompilationUnitRoot(), fullTree.GetCompilationUnitRoot()); } - [Fact] - [WorkItem("https://github.com/dotnet/roslyn/issues/74456")] + [Fact, WorkItem("https://github.com/dotnet/roslyn/issues/74456")] public void TestCollectionExpressionSpreadVsDeletingTopLevelBrace() { - var source = """ + var initialSource = """ namespace Example; public sealed class Program @@ -613,32 +612,40 @@ public void M2() } } """; - var tree = SyntaxFactory.ParseSyntaxTree(source); - Assert.Empty(tree.GetDiagnostics()); + var initialTree = SyntaxFactory.ParseSyntaxTree(initialSource); + + // Initial code is fully legal and should have no parse errors. + Assert.Empty(initialTree.GetDiagnostics()); + // Delete '{' (and end of line) before 'values[1] = 312;' const string valueSetterLine = "values[1] = 312;"; - var text = tree.GetText(); - var valueSetterLinePosition = source.IndexOf(valueSetterLine); - var lines = text.Lines; + var initialText = initialTree.GetText(); + var valueSetterLinePosition = initialSource.IndexOf(valueSetterLine); + var initialLines = initialText.Lines; - int valueSetterLineIndex = lines.IndexOf(valueSetterLinePosition); - var openBraceLine = text.Lines[valueSetterLineIndex - 1]; + int valueSetterLineIndex = initialLines.IndexOf(valueSetterLinePosition); + var openBraceLine = initialText.Lines[valueSetterLineIndex - 1]; Assert.EndsWith("{", openBraceLine.ToString()); - text = text.WithChanges(new TextChange(openBraceLine.SpanIncludingLineBreak, "")); - tree = tree.WithChangedText(text); - lines = text.Lines; - var closeBraceLine = text.Lines[valueSetterLineIndex]; + var withOpenBraceDeletedText = initialText.WithChanges(new TextChange(openBraceLine.SpanIncludingLineBreak, "")); + var withOpenBraceDeletedTree = initialTree.WithChangedText(withOpenBraceDeletedText); + + // Now delete '}' after 'values[1] = 312;'. This should result in no diagnostics. + // + // Note: because we deleted the end of line after the '{', the line that is now on the line where `values...` was + // will be the line that was originally after it (the } line). + var withOpenBraceDeletedLines = withOpenBraceDeletedText.Lines; + var closeBraceLine = withOpenBraceDeletedLines[valueSetterLineIndex]; Assert.EndsWith("}", closeBraceLine.ToString()); - text = text.WithChanges(new TextChange(closeBraceLine.SpanIncludingLineBreak, "")); - tree = tree.WithChangedText(text); + var withCloseBraceDeletedText = withOpenBraceDeletedText.WithChanges(new TextChange(closeBraceLine.SpanIncludingLineBreak, "")); + var withCloseBraceDeletedTree = withOpenBraceDeletedTree.WithChangedText(withCloseBraceDeletedText); - Assert.Empty(tree.GetDiagnostics()); + Assert.Empty(withCloseBraceDeletedTree.GetDiagnostics()); - var fullTree = SyntaxFactory.ParseSyntaxTree(text.ToString()); + var fullTree = SyntaxFactory.ParseSyntaxTree(withCloseBraceDeletedText.ToString()); Assert.Empty(fullTree.GetDiagnostics()); - WalkTreeAndVerify(tree.GetCompilationUnitRoot(), fullTree.GetCompilationUnitRoot()); + WalkTreeAndVerify(withCloseBraceDeletedTree.GetCompilationUnitRoot(), fullTree.GetCompilationUnitRoot()); } #region "Regression" From 80b11eeb5f74ea6d455e122d9383c7e7bc2cf484 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 12:27:31 -0700 Subject: [PATCH 04/44] Update tests --- .../IncrementalParsingTests.cs | 227 +++++++++++++++++- .../Test/Syntax/Parsing/ParsingTests.cs | 2 +- 2 files changed, 215 insertions(+), 14 deletions(-) diff --git a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs index b225d392486ad..352274a2b09c7 100644 --- a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs @@ -7,15 +7,19 @@ using System; using System.Collections.Immutable; using System.Linq; +using ICSharpCode.Decompiler.TypeSystem; +using System.Reflection; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.CSharp.Test.Utilities; using Microsoft.CodeAnalysis.Text; +using Newtonsoft.Json.Linq; using Roslyn.Test.Utilities; using Xunit; +using Xunit.Abstractions; namespace Microsoft.CodeAnalysis.CSharp.UnitTests { - public class IncrementalParsingTests : TestBase + public sealed class IncrementalParsingTests(ITestOutputHelper output) : ParsingTests(output) { private CSharpParseOptions GetOptions(string[] defines) { @@ -587,27 +591,24 @@ int LocalFunc() [Fact, WorkItem("https://github.com/dotnet/roslyn/issues/74456")] public void TestCollectionExpressionSpreadVsDeletingTopLevelBrace() { - var initialSource = """ - namespace Example; - - public sealed class Program + const string valueSetterLine = "x[1] = 312;"; + var initialSource = $$""" + public class Program { public void M2() { - const bool condition = true; - int[] values = [1, 2, 3]; - if (condition) + if (true) { { - if (condition) + if (true) { - values[1] = 312; + {{valueSetterLine}} } } } - if (condition) + if (true) { - values = [.. values]; + y = [.. z]; } } } @@ -618,7 +619,6 @@ public void M2() Assert.Empty(initialTree.GetDiagnostics()); // Delete '{' (and end of line) before 'values[1] = 312;' - const string valueSetterLine = "values[1] = 312;"; var initialText = initialTree.GetText(); var valueSetterLinePosition = initialSource.IndexOf(valueSetterLine); var initialLines = initialText.Lines; @@ -630,6 +630,207 @@ public void M2() var withOpenBraceDeletedText = initialText.WithChanges(new TextChange(openBraceLine.SpanIncludingLineBreak, "")); var withOpenBraceDeletedTree = initialTree.WithChangedText(withOpenBraceDeletedText); + // Deletion of the open brace causes the method body to close early with the close brace before the `if` + // statement. This will lead to a ton of cascading errors for what follows. In particular, the `[.. values]` + // will be parsed as a very broken attribute on an incomplete member. + { + UsingTree(withOpenBraceDeletedTree, + // (13,9): error CS1519: Invalid token 'if' in class, record, struct, or interface member declaration + // if (true) + Diagnostic(ErrorCode.ERR_InvalidMemberDecl, "if").WithArguments("if").WithLocation(13, 9), + // (13,13): error CS1031: Type expected + // if (true) + Diagnostic(ErrorCode.ERR_TypeExpected, "true").WithLocation(13, 13), + // (13,13): error CS8124: Tuple must contain at least two elements. + // if (true) + Diagnostic(ErrorCode.ERR_TupleTooFewElements, "true").WithLocation(13, 13), + // (13,13): error CS1026: ) expected + // if (true) + Diagnostic(ErrorCode.ERR_CloseParenExpected, "true").WithLocation(13, 13), + // (13,13): error CS1519: Invalid token 'true' in class, record, struct, or interface member declaration + // if (true) + Diagnostic(ErrorCode.ERR_InvalidMemberDecl, "true").WithArguments("true").WithLocation(13, 13), + // (15,15): error CS1519: Invalid token '=' in class, record, struct, or interface member declaration + // y = [.. z]; + Diagnostic(ErrorCode.ERR_InvalidMemberDecl, "=").WithArguments("=").WithLocation(15, 15), + // (15,15): error CS1519: Invalid token '=' in class, record, struct, or interface member declaration + // y = [.. z]; + Diagnostic(ErrorCode.ERR_InvalidMemberDecl, "=").WithArguments("=").WithLocation(15, 15), + // (15,18): error CS1001: Identifier expected + // y = [.. z]; + Diagnostic(ErrorCode.ERR_IdentifierExpected, "..").WithLocation(15, 18), + // (15,19): error CS1001: Identifier expected + // y = [.. z]; + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(15, 19), + // (15,23): error CS1519: Invalid token ';' in class, record, struct, or interface member declaration + // y = [.. z]; + Diagnostic(ErrorCode.ERR_InvalidMemberDecl, ";").WithArguments(";").WithLocation(15, 23), + // (17,5): error CS1022: Type or namespace definition, or end-of-file expected + // } + Diagnostic(ErrorCode.ERR_EOFExpected, "}").WithLocation(17, 5), + // (18,1): error CS1022: Type or namespace definition, or end-of-file expected + // } + Diagnostic(ErrorCode.ERR_EOFExpected, "}").WithLocation(18, 1)); + + N(SyntaxKind.CompilationUnit); + { + N(SyntaxKind.ClassDeclaration); + { + N(SyntaxKind.PublicKeyword); + N(SyntaxKind.ClassKeyword); + N(SyntaxKind.IdentifierToken, "Program"); + N(SyntaxKind.OpenBraceToken); + N(SyntaxKind.MethodDeclaration); + { + N(SyntaxKind.PublicKeyword); + N(SyntaxKind.PredefinedType); + { + N(SyntaxKind.VoidKeyword); + } + N(SyntaxKind.IdentifierToken, "M2"); + N(SyntaxKind.ParameterList); + { + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.CloseParenToken); + } + N(SyntaxKind.Block); + { + N(SyntaxKind.OpenBraceToken); + N(SyntaxKind.IfStatement); + { + N(SyntaxKind.IfKeyword); + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.TrueLiteralExpression); + { + N(SyntaxKind.TrueKeyword); + } + N(SyntaxKind.CloseParenToken); + N(SyntaxKind.Block); + { + N(SyntaxKind.OpenBraceToken); + N(SyntaxKind.Block); + { + N(SyntaxKind.OpenBraceToken); + N(SyntaxKind.IfStatement); + { + N(SyntaxKind.IfKeyword); + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.TrueLiteralExpression); + { + N(SyntaxKind.TrueKeyword); + } + N(SyntaxKind.CloseParenToken); + N(SyntaxKind.ExpressionStatement); + { + N(SyntaxKind.SimpleAssignmentExpression); + { + N(SyntaxKind.ElementAccessExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "x"); + } + N(SyntaxKind.BracketedArgumentList); + { + N(SyntaxKind.OpenBracketToken); + N(SyntaxKind.Argument); + { + N(SyntaxKind.NumericLiteralExpression); + { + N(SyntaxKind.NumericLiteralToken, "1"); + } + } + N(SyntaxKind.CloseBracketToken); + } + } + N(SyntaxKind.EqualsToken); + N(SyntaxKind.NumericLiteralExpression); + { + N(SyntaxKind.NumericLiteralToken, "312"); + } + } + N(SyntaxKind.SemicolonToken); + } + } + N(SyntaxKind.CloseBraceToken); + } + N(SyntaxKind.CloseBraceToken); + } + } + N(SyntaxKind.CloseBraceToken); + } + } + // Here is where we go off the rails. This corresponds to the `if (true) ...` part after the method + N(SyntaxKind.IncompleteMember); + { + N(SyntaxKind.TupleType); + { + N(SyntaxKind.OpenParenToken); + M(SyntaxKind.TupleElement); + { + M(SyntaxKind.IdentifierName); + { + M(SyntaxKind.IdentifierToken); + } + } + M(SyntaxKind.CommaToken); + M(SyntaxKind.TupleElement); + { + M(SyntaxKind.IdentifierName); + { + M(SyntaxKind.IdentifierToken); + } + } + M(SyntaxKind.CloseParenToken); + } + } + // this corresponds to 'y' in 'y = [.. z];' + N(SyntaxKind.IncompleteMember); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "y"); + } + } + // This corresponds to `[.. z]` which parser thinks is an attribute with an invalid dotted name. + N(SyntaxKind.IncompleteMember); + { + N(SyntaxKind.AttributeList); + { + N(SyntaxKind.OpenBracketToken); + N(SyntaxKind.Attribute); + { + N(SyntaxKind.QualifiedName); + { + N(SyntaxKind.QualifiedName); + { + M(SyntaxKind.IdentifierName); + { + M(SyntaxKind.IdentifierToken); + } + N(SyntaxKind.DotToken); + M(SyntaxKind.IdentifierName); + { + M(SyntaxKind.IdentifierToken); + } + } + N(SyntaxKind.DotToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "z"); + } + } + } + N(SyntaxKind.CloseBracketToken); + } + } + N(SyntaxKind.CloseBraceToken); + } + N(SyntaxKind.EndOfFileToken); + } + EOF(); + } + // Now delete '}' after 'values[1] = 312;'. This should result in no diagnostics. // // Note: because we deleted the end of line after the '{', the line that is now on the line where `values...` was diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs index 930ced6a0a2bc..f32350f84b5bf 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs @@ -34,7 +34,7 @@ public override void Dispose() VerifyEnumeratorConsumed(); } - private void VerifyEnumeratorConsumed() + protected void VerifyEnumeratorConsumed() { if (_treeEnumerator != null) { From 5a32172722a4a8273ef745a9f73122010d2a7737 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 13:24:32 -0700 Subject: [PATCH 05/44] Do not have lexer produce .. token --- .../CSharp/Portable/Parser/Blender.Reader.cs | 4 +- .../CSharp/Portable/Parser/Blender.cs | 3 - .../CSharp/Portable/Parser/LanguageParser.cs | 222 ++++++++---------- .../Parser/LanguageParser_Patterns.cs | 9 +- src/Compilers/CSharp/Portable/Parser/Lexer.cs | 28 ++- 5 files changed, 125 insertions(+), 141 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/Blender.Reader.cs b/src/Compilers/CSharp/Portable/Parser/Blender.Reader.cs index 4181b444e1c75..eb0a967120e9f 100644 --- a/src/Compilers/CSharp/Portable/Parser/Blender.Reader.cs +++ b/src/Compilers/CSharp/Portable/Parser/Blender.Reader.cs @@ -6,10 +6,7 @@ using System.Collections.Immutable; using System.Diagnostics; -using Microsoft.CodeAnalysis.CSharp.Symbols; -using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Text; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax { @@ -307,6 +304,7 @@ internal static bool IsFabricatedToken(SyntaxKind kind) case SyntaxKind.GreaterThanGreaterThanEqualsToken: case SyntaxKind.GreaterThanGreaterThanGreaterThanToken: case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: + case SyntaxKind.DotDotToken: return true; default: return SyntaxFacts.IsContextualKeyword(kind); diff --git a/src/Compilers/CSharp/Portable/Parser/Blender.cs b/src/Compilers/CSharp/Portable/Parser/Blender.cs index 0f5e154d0d7ec..064c234f940cd 100644 --- a/src/Compilers/CSharp/Portable/Parser/Blender.cs +++ b/src/Compilers/CSharp/Portable/Parser/Blender.cs @@ -8,10 +8,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; -using Microsoft.CodeAnalysis.CSharp.Symbols; -using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Text; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax { diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index f3db0e485c25d..60bd84a7edd85 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -3235,7 +3235,6 @@ private bool IsFieldDeclaration(bool isEvent, bool isGlobalScriptLevel) { case SyntaxKind.DotToken: // Goo. explicit case SyntaxKind.ColonColonToken: // Goo:: explicit - case SyntaxKind.DotDotToken: // Goo.. explicit case SyntaxKind.LessThanToken: // Goo< explicit or generic method case SyntaxKind.OpenBraceToken: // Goo { property case SyntaxKind.EqualsGreaterThanToken: // Goo => property @@ -3563,12 +3562,12 @@ private ConversionOperatorDeclarationSyntax TryParseConversionOperatorDeclaratio IsMakingProgress(ref lastTokenPosition, assertIfFalse: true); ScanNamedTypePart(); - if (IsDotOrColonColonOrDotDot() || + if (IsDotOrColonColon() || (IsMakingProgress(ref lastTokenPosition, assertIfFalse: false) && this.CurrentToken.Kind != SyntaxKind.OpenParenToken)) { haveExplicitInterfaceName = true; - if (IsDotOrColonColonOrDotDot()) + if (IsDotOrColonColon()) { separatorKind = this.CurrentToken.Kind; EatToken(); @@ -3593,7 +3592,7 @@ private ConversionOperatorDeclarationSyntax TryParseConversionOperatorDeclaratio bool possibleConversion; if (this.CurrentToken.Kind != SyntaxKind.OperatorKeyword || - (haveExplicitInterfaceName && separatorKind is not (SyntaxKind.DotToken or SyntaxKind.DotDotToken))) + (haveExplicitInterfaceName && separatorKind is not SyntaxKind.DotToken)) { possibleConversion = false; } @@ -3718,8 +3717,8 @@ ExplicitInterfaceSpecifierSyntax tryParseExplicitInterfaceSpecifier() int lastTokenPosition = -1; IsMakingProgress(ref lastTokenPosition, assertIfFalse: true); ScanNamedTypePart(); - isPartOfInterfaceName = IsDotOrColonColonOrDotDot() || - (IsMakingProgress(ref lastTokenPosition, assertIfFalse: false) && this.CurrentToken.Kind != SyntaxKind.OpenParenToken); + isPartOfInterfaceName = IsDotOrColonColon() || + (IsMakingProgress(ref lastTokenPosition, assertIfFalse: false) && this.CurrentToken.Kind != SyntaxKind.OpenParenToken); } } @@ -6425,7 +6424,7 @@ private void ParseMemberName( using (GetDisposableResetPoint(resetOnDispose: true)) { ScanNamedTypePart(); - isMemberName = !IsDotOrColonColonOrDotDot(); + isMemberName = !IsDotOrColonColon(); } if (isMemberName) @@ -6531,18 +6530,9 @@ private void AccumulateExplicitInterfaceName(ref NameSyntax explicitInterfaceNam explicitInterfaceName = this.ParseSimpleName(NameOptions.InTypeList); // Now, get the next separator. - if (this.CurrentToken.Kind == SyntaxKind.DotDotToken) - { - // Error recovery as in ParseQualifiedNameRight. If we have `X..Y` break that into `X..Y` - separator = this.EatToken(); - explicitInterfaceName = RecoverFromDotDot(explicitInterfaceName, ref separator); - } - else - { - separator = this.CurrentToken.Kind == SyntaxKind.ColonColonToken - ? this.EatToken() // fine after the first identifier - : this.EatToken(SyntaxKind.DotToken); - } + separator = this.CurrentToken.Kind == SyntaxKind.ColonColonToken + ? this.EatToken() // fine after the first identifier + : this.EatToken(SyntaxKind.DotToken); } else { @@ -6559,12 +6549,6 @@ private void AccumulateExplicitInterfaceName(ref NameSyntax explicitInterfaceNam separator = this.AddError(separator, ErrorCode.ERR_UnexpectedAliasedName); separator = this.ConvertToMissingWithTrailingTrivia(separator, SyntaxKind.DotToken); } - else if (this.CurrentToken.Kind == SyntaxKind.DotDotToken) - { - // Error recovery as in ParseQualifiedNameRight. If we have `X..Y` break that into `X..Y` - separator = this.EatToken(); - explicitInterfaceName = RecoverFromDotDot(explicitInterfaceName, ref separator); - } else { separator = this.EatToken(SyntaxKind.DotToken); @@ -6617,7 +6601,7 @@ private bool IsOperatorStart(out ExplicitInterfaceSpecifierSyntax explicitInterf // If we have part of the interface name, but no dot before the operator token, then // for the purpose of error recovery, treat this as an operator start with a // missing dot token. - isPartOfInterfaceName = IsDotOrColonColonOrDotDot() || IsOperatorKeyword(); + isPartOfInterfaceName = IsDotOrColonColon() || IsOperatorKeyword(); } } @@ -6674,7 +6658,7 @@ private NameSyntax ParseQualifiedName(NameOptions options = NameOptions.None) NameSyntax name = this.ParseAliasQualifiedName(options); // Handle .. tokens for error recovery purposes. - while (IsDotOrColonColonOrDotDot()) + while (IsDotOrColonColon()) { if (this.PeekToken(1).Kind == SyntaxKind.ThisKeyword) { @@ -6688,26 +6672,18 @@ private NameSyntax ParseQualifiedName(NameOptions options = NameOptions.None) return name; } - private bool IsDotOrColonColonOrDotDot() - { - return this.IsDotOrColonColon() || this.CurrentToken.Kind == SyntaxKind.DotDotToken; - } - private NameSyntax ParseQualifiedNameRight( NameOptions options, NameSyntax left, SyntaxToken separator) { - Debug.Assert(separator.Kind is SyntaxKind.DotToken or SyntaxKind.DotDotToken or SyntaxKind.ColonColonToken); + Debug.Assert(separator.Kind is SyntaxKind.DotToken or SyntaxKind.ColonColonToken); var right = this.ParseSimpleName(options); switch (separator.Kind) { case SyntaxKind.DotToken: return _syntaxFactory.QualifiedName(left, separator, right); - case SyntaxKind.DotDotToken: - // Error recovery. If we have `X..Y` break that into `X..Y` - return _syntaxFactory.QualifiedName(RecoverFromDotDot(left, ref separator), separator, right); case SyntaxKind.ColonColonToken: if (left.Kind != SyntaxKind.IdentifierName) @@ -6741,15 +6717,15 @@ private NameSyntax ParseQualifiedNameRight( } } - private NameSyntax RecoverFromDotDot(NameSyntax left, ref SyntaxToken separator) - { - Debug.Assert(separator.Kind == SyntaxKind.DotDotToken); + //private NameSyntax RecoverFromDotDot(NameSyntax left, ref SyntaxToken separator) + //{ + // Debug.Assert(separator.Kind == SyntaxKind.DotDotToken); - var leftDot = SyntaxFactory.Token(separator.LeadingTrivia.Node, SyntaxKind.DotToken, null); - var missingName = this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_IdentifierExpected); - separator = SyntaxFactory.Token(null, SyntaxKind.DotToken, separator.TrailingTrivia.Node); - return _syntaxFactory.QualifiedName(left, leftDot, missingName); - } + // var leftDot = SyntaxFactory.Token(separator.LeadingTrivia.Node, SyntaxKind.DotToken, null); + // var missingName = this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_IdentifierExpected); + // separator = SyntaxFactory.Token(null, SyntaxKind.DotToken, separator.TrailingTrivia.Node); + // return _syntaxFactory.QualifiedName(left, leftDot, missingName); + //} private SyntaxToken ConvertToMissingWithTrailingTrivia(SyntaxToken token, SyntaxKind expectedKind) { @@ -7893,7 +7869,7 @@ or SyntaxKind.MinusMinusToken isCollectionExpression = isCollectionExpression || IsExpectedBinaryOperator(this.CurrentToken.Kind) || IsExpectedAssignmentOperator(this.CurrentToken.Kind) - || this.CurrentToken.Kind is SyntaxKind.DotDotToken + || IsAtDotDotToken() || (this.CurrentToken.ContextualKind is SyntaxKind.SwitchKeyword or SyntaxKind.WithKeyword && this.PeekToken(1).Kind is SyntaxKind.OpenBraceToken); if (!isCollectionExpression && @@ -10616,10 +10592,11 @@ private bool IsPossibleExpression(bool allowBinaryExpressions, bool allowAssignm case SyntaxKind.ColonColonToken: // bad aliased name case SyntaxKind.ThrowKeyword: case SyntaxKind.StackAllocKeyword: - case SyntaxKind.DotDotToken: case SyntaxKind.RefKeyword: case SyntaxKind.OpenBracketToken: // attributes on a lambda, or a collection expression. return true; + case SyntaxKind.DotToken when IsAtDotDotToken(): + return true; case SyntaxKind.StaticKeyword: return IsPossibleAnonymousMethodExpression() || IsPossibleLambdaExpression(Precedence.Expression); case SyntaxKind.IdentifierToken: @@ -10962,23 +10939,14 @@ private ExpressionSyntax ParseSubExpressionCore(Precedence precedence) var operand = this.ParseSubExpression(newPrecedence); leftOperand = _syntaxFactory.PrefixUnaryExpression(opKind, opToken, operand); } - else if (tk == SyntaxKind.DotDotToken) + else if (IsAtDotDotToken()) { - // Operator ".." here can either be a prefix unary operator or a stand alone empty range: - var opToken = this.EatToken(); - newPrecedence = GetPrecedence(SyntaxKind.RangeExpression); - - ExpressionSyntax rightOperand; - if (CanStartExpression()) - { - rightOperand = this.ParseSubExpression(newPrecedence); - } - else - { - rightOperand = null; - } - - leftOperand = _syntaxFactory.RangeExpression(leftOperand: null, opToken, rightOperand); + leftOperand = _syntaxFactory.RangeExpression( + leftOperand: null, + this.EatDotDotToken(), + CanStartExpression() + ? this.ParseSubExpression(GetPrecedence(SyntaxKind.RangeExpression)) + : null); } else if (IsAwaitExpression()) { @@ -11034,6 +11002,8 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, // to see if it may need a similar look-ahead check to determine if something is a collection expression versus // an attribute. + int tokensToCombine = 1; + if (IsExpectedBinaryOperator(tk)) { opKind = SyntaxFacts.GetBinaryExpression(tk); @@ -11043,10 +11013,6 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, opKind = SyntaxFacts.GetAssignmentExpression(tk); isAssignmentOperator = true; } - else if (tk == SyntaxKind.DotDotToken) - { - opKind = SyntaxKind.RangeExpression; - } else if (tk == SyntaxKind.SwitchKeyword && this.PeekToken(1).Kind == SyntaxKind.OpenBraceToken) { opKind = SyntaxKind.SwitchExpression; @@ -11055,52 +11021,57 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, { opKind = SyntaxKind.WithExpression; } + // Check for .. + else if (IsAtDotDotToken()) + { + tokensToCombine = 2; + opKind = SyntaxKind.RangeExpression; + } else { break; } - var newPrecedence = GetPrecedence(opKind); - - // check for >>, >>=, >>> or >>>= - int tokensToCombine = 1; - if (tk == SyntaxKind.GreaterThanToken - && this.PeekToken(1).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken - && NoTriviaBetween(this.CurrentToken, this.PeekToken(1))) // check to see if they really are adjacent { - if (this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken) + // check for >>, >>=, >>> or >>>= + if (tk == SyntaxKind.GreaterThanToken + && this.PeekToken(1).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken + && NoTriviaBetween(this.CurrentToken, this.PeekToken(1))) // check to see if they really are adjacent { - if (this.PeekToken(2).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken - && NoTriviaBetween(this.PeekToken(1), this.PeekToken(2))) // check to see if they really are adjacent + if (this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken) { - if (this.PeekToken(2).Kind == SyntaxKind.GreaterThanToken) + if (this.PeekToken(2).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken + && NoTriviaBetween(this.PeekToken(1), this.PeekToken(2))) // check to see if they really are adjacent { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken); + if (this.PeekToken(2).Kind == SyntaxKind.GreaterThanToken) + { + opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken); + } + else + { + opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); + isAssignmentOperator = true; + } + + tokensToCombine = 3; } else { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); - isAssignmentOperator = true; + opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); + tokensToCombine = 2; } - - tokensToCombine = 3; } else { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); + opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); + isAssignmentOperator = true; tokensToCombine = 2; } } - else - { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); - isAssignmentOperator = true; - tokensToCombine = 2; - } - - newPrecedence = GetPrecedence(opKind); } + var newPrecedence = GetPrecedence(opKind); + // Check the precedence to see if we should "take" this operator if (newPrecedence < precedence) { @@ -11137,9 +11108,16 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, // Combine tokens into a single token if needed if (tokensToCombine == 2) { - var opToken2 = this.EatToken(); - var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken; - opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); + if (tk == SyntaxKind.DotToken) + { + opToken = this.EatDotDotToken(); + } + else + { + var opToken2 = this.EatToken(); + var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken; + opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); + } } else if (tokensToCombine == 3) { @@ -11190,23 +11168,14 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, { leftOperand = ParseWithExpression(leftOperand, opToken); } - else if (tk == SyntaxKind.DotDotToken) + else if (opKind == SyntaxKind.RangeExpression) { - // Operator ".." here can either be a binary or a postfix unary operator: - Debug.Assert(opKind == SyntaxKind.RangeExpression); - - ExpressionSyntax rightOperand; - if (CanStartExpression()) - { - newPrecedence = GetPrecedence(opKind); - rightOperand = this.ParseSubExpression(newPrecedence); - } - else - { - rightOperand = null; - } - - leftOperand = _syntaxFactory.RangeExpression(leftOperand, opToken, rightOperand); + leftOperand = _syntaxFactory.RangeExpression( + leftOperand, + opToken, + CanStartExpression() + ? this.ParseSubExpression(Precedence.Range) + : null); } else { @@ -11319,6 +11288,22 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) } } + public bool IsAtDotDotToken() + { + var token1 = this.CurrentToken; + var token2 = this.PeekToken(1); + return token1.Kind == SyntaxKind.DotToken && token2.Kind == SyntaxKind.DotToken && NoTriviaBetween(token1, token2); + } + + public SyntaxToken EatDotDotToken() + { + Debug.Assert(IsAtDotDotToken()); + var token1 = this.EatToken(); + var token2 = this.EatToken(); + + return SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + } + private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) { var scopedKeyword = isScoped @@ -12341,8 +12326,11 @@ private bool ScanCast(bool forPattern = false) SyntaxKind.PlusToken or SyntaxKind.MinusToken or SyntaxKind.AmpersandToken or - SyntaxKind.AsteriskToken or - SyntaxKind.DotDotToken + SyntaxKind.AsteriskToken + => true, + + // `(X)..` must be a cast of a range expression, not a member access of some arbitrary expression. + SyntaxKind.DotToken when IsAtDotDotToken() => true, var tk @@ -12595,7 +12583,6 @@ private static bool CanFollowCast(SyntaxKind kind) case SyntaxKind.EndOfFileToken: case SyntaxKind.SwitchKeyword: case SyntaxKind.EqualsGreaterThanToken: - case SyntaxKind.DotDotToken: return false; default: return true; @@ -12657,12 +12644,9 @@ private bool IsPossibleCollectionElement() private CollectionElementSyntax ParseCollectionElement() { - var dotDotToken = this.TryEatToken(SyntaxKind.DotDotToken); - if (dotDotToken != null) - return _syntaxFactory.SpreadElement(dotDotToken, this.ParseExpressionCore()); - - var expression = this.ParseExpressionCore(); - return _syntaxFactory.ExpressionElement(expression); + return IsAtDotDotToken() + ? _syntaxFactory.SpreadElement(this.EatDotDotToken(), this.ParseExpressionCore()) + : _syntaxFactory.ExpressionElement(this.ParseExpressionCore()); } private bool IsAnonymousType() diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser_Patterns.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser_Patterns.cs index ad961242a9386..712ddc1633420 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser_Patterns.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser_Patterns.cs @@ -193,9 +193,12 @@ private PatternSyntax ParsePrimaryPattern(Precedence precedence, bool afterIs, b { case SyntaxKind.OpenBracketToken: return this.ParseListPattern(inSwitchArmPattern); - case SyntaxKind.DotDotToken: - return _syntaxFactory.SlicePattern(EatToken(), - IsPossibleSubpatternElement() ? ParsePattern(precedence, afterIs: false, inSwitchArmPattern) : null); + case SyntaxKind.DotToken when IsAtDotDotToken(): + return _syntaxFactory.SlicePattern( + EatDotDotToken(), + IsPossibleSubpatternElement() + ? ParsePattern(precedence, afterIs: false, inSwitchArmPattern) + : null); case SyntaxKind.LessThanToken: case SyntaxKind.LessThanEqualsToken: case SyntaxKind.GreaterThanToken: diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs index bcb3d9296d2ad..3b0e003fcec08 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer.cs @@ -461,23 +461,25 @@ private void ScanSyntaxToken(ref TokenInfo info) case '.': if (!this.ScanNumericLiteral(ref info)) { + // Consume the dot. + // + // Note: Similar to `>>` we do not consume a `..` as a single token in the lexer. Instead it is + // synthesized by the parser on demand. This ensures in incremental scenarios as well we treat this + // the combined character as something synthesized and something to be broken apart in case we want + // different treatments later on. TextWindow.AdvanceChar(); - if (TextWindow.TryAdvance('.')) + if (TextWindow.PeekChar(0) == '.' && + TextWindow.PeekChar(1) == '.') { - if (TextWindow.PeekChar() == '.') - { - // Triple-dot: explicitly reject this, to allow triple-dot - // to be added to the language without a breaking change. - // (without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2) - this.AddError(ErrorCode.ERR_TripleDotNotAllowed); - } + // Triple-dot: explicitly reject this, to allow triple-dot + // to be added to the language without a breaking change. + // (without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2) - info.Kind = SyntaxKind.DotDotToken; - } - else - { - info.Kind = SyntaxKind.DotToken; + // We're at the 2nd dot in the ... So add '1' to the position to report on the 3rd dot. + this.AddError(position: TextWindow.Position + 1, width: 1, ErrorCode.ERR_TripleDotNotAllowed); } + + info.Kind = SyntaxKind.DotToken; } break; From ade242ce336eb448b148963b889749dbcfd95867 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 13:33:11 -0700 Subject: [PATCH 06/44] Move error --- .../CSharp/Portable/Parser/LanguageParser.cs | 13 ++++++++++++- src/Compilers/CSharp/Portable/Parser/Lexer.cs | 18 ------------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 60bd84a7edd85..39df12029abb5 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11301,7 +11301,18 @@ public SyntaxToken EatDotDotToken() var token1 = this.EatToken(); var token2 = this.EatToken(); - return SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + var dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + + // Triple-dot: explicitly reject this, to allow triple-dot to be added to the language without a breaking + // change. Without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2 + if (this.CurrentToken.Kind == SyntaxKind.DotToken && + NoTriviaBetween(token2, this.CurrentToken)) + { + var trailingDot = this.AddError(this.EatToken(), ErrorCode.ERR_TripleDotNotAllowed); + return AddTrailingSkippedSyntax(dotDotToken, trailingDot); + } + + return dotDotToken; } private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs index 3b0e003fcec08..91afda2193c07 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer.cs @@ -461,24 +461,6 @@ private void ScanSyntaxToken(ref TokenInfo info) case '.': if (!this.ScanNumericLiteral(ref info)) { - // Consume the dot. - // - // Note: Similar to `>>` we do not consume a `..` as a single token in the lexer. Instead it is - // synthesized by the parser on demand. This ensures in incremental scenarios as well we treat this - // the combined character as something synthesized and something to be broken apart in case we want - // different treatments later on. - TextWindow.AdvanceChar(); - if (TextWindow.PeekChar(0) == '.' && - TextWindow.PeekChar(1) == '.') - { - // Triple-dot: explicitly reject this, to allow triple-dot - // to be added to the language without a breaking change. - // (without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2) - - // We're at the 2nd dot in the ... So add '1' to the position to report on the 3rd dot. - this.AddError(position: TextWindow.Position + 1, width: 1, ErrorCode.ERR_TripleDotNotAllowed); - } - info.Kind = SyntaxKind.DotToken; } break; From 98016ccabb2f34ac5a6917661c63b93d83079589 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 13:35:28 -0700 Subject: [PATCH 07/44] Docs --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 39df12029abb5..c5256f3053ca0 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11288,6 +11288,7 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) } } + /// Check if we're currently at a .. sequence that can then be parsed out as a . public bool IsAtDotDotToken() { var token1 = this.CurrentToken; @@ -11295,6 +11296,9 @@ public bool IsAtDotDotToken() return token1.Kind == SyntaxKind.DotToken && token2.Kind == SyntaxKind.DotToken && NoTriviaBetween(token1, token2); } + /// Consume the next two tokens as a . Note: if three dot tokens + /// are in a row, the third will be consumed as skipped trivia to disallow any usage of ... in the + /// language. public SyntaxToken EatDotDotToken() { Debug.Assert(IsAtDotDotToken()); From 3631afb222161be5196cb8a725ef084739e7e460 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 13:42:52 -0700 Subject: [PATCH 08/44] Advance --- src/Compilers/CSharp/Portable/Parser/Lexer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs index 91afda2193c07..e37fbfa30bdef 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer.cs @@ -461,6 +461,7 @@ private void ScanSyntaxToken(ref TokenInfo info) case '.': if (!this.ScanNumericLiteral(ref info)) { + TextWindow.AdvanceChar(); info.Kind = SyntaxKind.DotToken; } break; From 24d5ef7ac0808013255b348dd2a9ac5eb20d8824 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 13:54:35 -0700 Subject: [PATCH 09/44] Place at the same locaiton --- .../CSharp/Portable/Parser/LanguageParser.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index c5256f3053ca0..b3fedc5896f1a 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11297,8 +11297,7 @@ public bool IsAtDotDotToken() } /// Consume the next two tokens as a . Note: if three dot tokens - /// are in a row, the third will be consumed as skipped trivia to disallow any usage of ... in the - /// language. + /// are in a row, an error will be placed on the .. token to say that is illegal. public SyntaxToken EatDotDotToken() { Debug.Assert(IsAtDotDotToken()); @@ -11309,14 +11308,13 @@ public SyntaxToken EatDotDotToken() // Triple-dot: explicitly reject this, to allow triple-dot to be added to the language without a breaking // change. Without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2 - if (this.CurrentToken.Kind == SyntaxKind.DotToken && - NoTriviaBetween(token2, this.CurrentToken)) + if (this.CurrentToken.Kind != SyntaxKind.DotToken || + !NoTriviaBetween(token2, this.CurrentToken)) { - var trailingDot = this.AddError(this.EatToken(), ErrorCode.ERR_TripleDotNotAllowed); - return AddTrailingSkippedSyntax(dotDotToken, trailingDot); + return dotDotToken; } - return dotDotToken; + return this.AddError(dotDotToken, offset: dotDotToken.GetLeadingTriviaWidth(), length: 0, ErrorCode.ERR_TripleDotNotAllowed); } private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) From 756efc023fe0e516d25584c19650b9b5655b4a4d Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 14:11:51 -0700 Subject: [PATCH 10/44] Cleanup and consistency --- .../CSharp/Portable/Parser/LanguageParser.cs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index b3fedc5896f1a..857d86dd3c81e 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11110,13 +11110,16 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, { if (tk == SyntaxKind.DotToken) { - opToken = this.EatDotDotToken(); + opToken = this.EatDotDotToken(opToken, this.EatToken()); } else { - var opToken2 = this.EatToken(); - var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken; - opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); + opToken = SyntaxFactory.Token( + opToken.GetLeadingTrivia(), + this.CurrentToken.Kind == SyntaxKind.GreaterThanToken + ? SyntaxKind.GreaterThanGreaterThanToken + : SyntaxKind.GreaterThanGreaterThanEqualsToken, + this.EatToken().GetTrailingTrivia()); } } else if (tokensToCombine == 3) @@ -11301,8 +11304,13 @@ public bool IsAtDotDotToken() public SyntaxToken EatDotDotToken() { Debug.Assert(IsAtDotDotToken()); - var token1 = this.EatToken(); - var token2 = this.EatToken(); + return EatDotDotToken(this.EatToken(), this.EatToken()); + } + + private SyntaxToken EatDotDotToken(SyntaxToken token1, SyntaxToken token2) + { + Debug.Assert(token1.Kind == SyntaxKind.DotToken); + Debug.Assert(token2.Kind == SyntaxKind.DotToken); var dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); @@ -11636,7 +11644,7 @@ private ExpressionSyntax ParsePostFixExpression(ExpressionSyntax expr) expr = _syntaxFactory.MemberAccessExpression(SyntaxKind.PointerMemberAccessExpression, expr, this.EatToken(), this.ParseSimpleName(NameOptions.InExpression)); continue; - case SyntaxKind.DotToken: + case SyntaxKind.DotToken when !IsAtDotDotToken(): // if we have the error situation: // // expr. From 1123f410925053bae6990b542eb37f22c9dc65e1 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 14:34:16 -0700 Subject: [PATCH 11/44] Break apart numerics --- .../CSharp/Portable/Parser/LanguageParser.cs | 102 ++++++++++++------ 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 857d86dd3c81e..ad7734ac31b7d 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11085,37 +11085,24 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, } // We'll "take" this operator, as precedence is tentatively OK. - var opToken = this.EatContextualToken(tk); - var leftPrecedence = GetPrecedence(leftOperand.Kind); - if (newPrecedence > leftPrecedence) - { - // Normally, a left operand with a looser precedence will consume all right operands that - // have a tighter precedence. For example, in the expression `a + b * c`, the `* c` part - // will be consumed as part of the right operand of the addition. However, there are a - // few circumstances in which a tighter precedence is not consumed: that occurs when the - // left hand operator does not have an expression as its right operand. This occurs for - // the is-type operator and the is-pattern operator. Source text such as - // `a is {} + b` should produce a syntax error, as parsing the `+` with an `is` - // expression as its left operand would be a precedence inversion. Similarly, it occurs - // with an anonymous method expression or a lambda expression with a block body. No - // further parsing will find a way to fix things up, so we accept the operator but issue - // a diagnostic. - ErrorCode errorCode = leftOperand.Kind == SyntaxKind.IsPatternExpression ? ErrorCode.ERR_UnexpectedToken : ErrorCode.WRN_PrecedenceInversion; - opToken = this.AddError(opToken, errorCode, opToken.Text); - } + SyntaxToken opToken; // Combine tokens into a single token if needed - if (tokensToCombine == 2) + if (tokensToCombine == 1) + { + opToken = this.EatContextualToken(tk); + } + else if (tokensToCombine == 2) { if (tk == SyntaxKind.DotToken) { - opToken = this.EatDotDotToken(opToken, this.EatToken()); + opToken = this.EatDotDotToken(); } else { opToken = SyntaxFactory.Token( - opToken.GetLeadingTrivia(), + this.EatContextualToken(tk).GetLeadingTrivia(), this.CurrentToken.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken, @@ -11124,17 +11111,36 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, } else if (tokensToCombine == 3) { + var token1 = this.EatContextualToken(tk); var opToken2 = this.EatToken(); Debug.Assert(opToken2.Kind == SyntaxKind.GreaterThanToken); opToken2 = this.EatToken(); var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; - opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); + opToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); } - else if (tokensToCombine != 1) + else { throw ExceptionUtilities.UnexpectedValue(tokensToCombine); } + var leftPrecedence = GetPrecedence(leftOperand.Kind); + if (newPrecedence > leftPrecedence) + { + // Normally, a left operand with a looser precedence will consume all right operands that + // have a tighter precedence. For example, in the expression `a + b * c`, the `* c` part + // will be consumed as part of the right operand of the addition. However, there are a + // few circumstances in which a tighter precedence is not consumed: that occurs when the + // left hand operator does not have an expression as its right operand. This occurs for + // the is-type operator and the is-pattern operator. Source text such as + // `a is {} + b` should produce a syntax error, as parsing the `+` with an `is` + // expression as its left operand would be a precedence inversion. Similarly, it occurs + // with an anonymous method expression or a lambda expression with a block body. No + // further parsing will find a way to fix things up, so we accept the operator but issue + // a diagnostic. + ErrorCode errorCode = leftOperand.Kind == SyntaxKind.IsPatternExpression ? ErrorCode.ERR_UnexpectedToken : ErrorCode.WRN_PrecedenceInversion; + opToken = this.AddError(opToken, errorCode, opToken.Text); + } + if (opKind == SyntaxKind.AsExpression) { var type = this.ParseType(ParseTypeMode.AsExpression); @@ -11295,24 +11301,58 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) public bool IsAtDotDotToken() { var token1 = this.CurrentToken; + if (token1.Kind != SyntaxKind.DotToken) + return false; + var token2 = this.PeekToken(1); - return token1.Kind == SyntaxKind.DotToken && token2.Kind == SyntaxKind.DotToken && NoTriviaBetween(token1, token2); + if (!NoTriviaBetween(token1, token2)) + return false; + + if (token2.Kind == SyntaxKind.DotToken) + { + // .. + // This is definitely a dot dot token. + return NoTriviaBetween(token1, token2); + } + + // ..Num + // + if (IsNumericLiteralStartingWithDot(token2)) + return true; + + return false; } + private static bool IsNumericLiteralStartingWithDot(SyntaxToken token) + => token is { Kind: SyntaxKind.NumericLiteralToken, Text: ['.', >= '0' and <= '9', ..] }; + /// Consume the next two tokens as a . Note: if three dot tokens /// are in a row, an error will be placed on the .. token to say that is illegal. public SyntaxToken EatDotDotToken() { Debug.Assert(IsAtDotDotToken()); - return EatDotDotToken(this.EatToken(), this.EatToken()); - } + var token1 = this.EatToken(); - private SyntaxToken EatDotDotToken(SyntaxToken token1, SyntaxToken token2) - { - Debug.Assert(token1.Kind == SyntaxKind.DotToken); - Debug.Assert(token2.Kind == SyntaxKind.DotToken); + var beforeSecondTokenPosition = this.lexer.TextWindow.Position; + var token2 = this.EatToken(); - var dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + SyntaxToken dotDotToken; + if (token2.Kind == SyntaxKind.DotToken) + { + dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + } + else if (IsNumericLiteralStartingWithDot(token2)) + { + // ..Num + + // Combine the `..` into one token, And reset the lexer to before 'Num' so it can lex it out. + dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, trailing: null); + this.lexer.Reset(beforeSecondTokenPosition + 1, this.lexer.Directives); + } + else + { + throw ExceptionUtilities.Unreachable(); + } // Triple-dot: explicitly reject this, to allow triple-dot to be added to the language without a breaking // change. Without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2 From 18692caee3dcefe4dab13ec5fd00451a2d58e4fa Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 16 Oct 2024 14:39:37 -0700 Subject: [PATCH 12/44] in progress --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- src/Compilers/CSharp/Portable/Parser/SyntaxParser.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index ad7734ac31b7d..3cb8e0963b8ee 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11339,7 +11339,7 @@ public SyntaxToken EatDotDotToken() SyntaxToken dotDotToken; if (token2.Kind == SyntaxKind.DotToken) { - dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, this.EatToken().GetTrailingTrivia()); } else if (IsNumericLiteralStartingWithDot(token2)) { diff --git a/src/Compilers/CSharp/Portable/Parser/SyntaxParser.cs b/src/Compilers/CSharp/Portable/Parser/SyntaxParser.cs index 9f499cc917a49..71b2a9c52bc36 100644 --- a/src/Compilers/CSharp/Portable/Parser/SyntaxParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/SyntaxParser.cs @@ -317,7 +317,7 @@ protected SyntaxToken CurrentToken { get { - return _currentToken ?? (_currentToken = this.FetchCurrentToken()); + return _currentToken ??= this.FetchCurrentToken(); } } From 5181f4702b651b83546a54237b7467ba37cdf8b1 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 10:36:33 -0700 Subject: [PATCH 13/44] Lexer work --- src/Compilers/CSharp/Portable/Parser/Lexer.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs index e37fbfa30bdef..62be0e60197f3 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer.cs @@ -459,11 +459,39 @@ private void ScanSyntaxToken(ref TokenInfo info) break; case '.': + if (this.TextWindow.PeekChar(1) is >= '0' and <= '9') + { + var atDotPosition = this.TextWindow.Position; + if (atDotPosition >= 1 && + atDotPosition == this.TextWindow.LexemeStartPosition) + { + // we have something like .0 this could be a fp number *except* the case where we have `..0` + // in that case, we want two dots followed by an integer (which will be treated as a range expression). + // + // Move back one space to see what's before this dot. + + this.TextWindow.Reset(atDotPosition - 1); + var priorCharacterIsDot = this.TextWindow.PeekChar() is '.'; + this.TextWindow.Reset(atDotPosition); + + if (priorCharacterIsDot) + { + // We have two dots in a row. Treat the second dot as a dot, not the start of a number literal. + TextWindow.AdvanceChar(); + info.Kind = SyntaxKind.DotToken; + break; + } + + // Fall through naturally and scan the number out as a floating point number. + } + } + if (!this.ScanNumericLiteral(ref info)) { TextWindow.AdvanceChar(); info.Kind = SyntaxKind.DotToken; } + break; case ',': From 35885675893727a86c4235d69082ded50e3081cf Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 10:39:41 -0700 Subject: [PATCH 14/44] restore --- .../CSharp/Portable/Parser/LanguageParser.cs | 56 +------------------ 1 file changed, 3 insertions(+), 53 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 3cb8e0963b8ee..23816b3b15f25 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -6717,16 +6717,6 @@ private NameSyntax ParseQualifiedNameRight( } } - //private NameSyntax RecoverFromDotDot(NameSyntax left, ref SyntaxToken separator) - //{ - // Debug.Assert(separator.Kind == SyntaxKind.DotDotToken); - - // var leftDot = SyntaxFactory.Token(separator.LeadingTrivia.Node, SyntaxKind.DotToken, null); - // var missingName = this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_IdentifierExpected); - // separator = SyntaxFactory.Token(null, SyntaxKind.DotToken, separator.TrailingTrivia.Node); - // return _syntaxFactory.QualifiedName(left, leftDot, missingName); - //} - private SyntaxToken ConvertToMissingWithTrailingTrivia(SyntaxToken token, SyntaxKind expectedKind) { var newToken = SyntaxFactory.MissingToken(expectedKind); @@ -11305,22 +11295,10 @@ public bool IsAtDotDotToken() return false; var token2 = this.PeekToken(1); - if (!NoTriviaBetween(token1, token2)) + if (token2.Kind != SyntaxKind.DotToken) return false; - if (token2.Kind == SyntaxKind.DotToken) - { - // .. - // This is definitely a dot dot token. - return NoTriviaBetween(token1, token2); - } - - // ..Num - // - if (IsNumericLiteralStartingWithDot(token2)) - return true; - - return false; + return NoTriviaBetween(token1, token2)) } private static bool IsNumericLiteralStartingWithDot(SyntaxToken token) @@ -11332,37 +11310,9 @@ public SyntaxToken EatDotDotToken() { Debug.Assert(IsAtDotDotToken()); var token1 = this.EatToken(); - - var beforeSecondTokenPosition = this.lexer.TextWindow.Position; var token2 = this.EatToken(); - SyntaxToken dotDotToken; - if (token2.Kind == SyntaxKind.DotToken) - { - dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, this.EatToken().GetTrailingTrivia()); - } - else if (IsNumericLiteralStartingWithDot(token2)) - { - // ..Num - - // Combine the `..` into one token, And reset the lexer to before 'Num' so it can lex it out. - dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, trailing: null); - this.lexer.Reset(beforeSecondTokenPosition + 1, this.lexer.Directives); - } - else - { - throw ExceptionUtilities.Unreachable(); - } - - // Triple-dot: explicitly reject this, to allow triple-dot to be added to the language without a breaking - // change. Without this, 0...2 would parse as (0)..(.2), i.e. a range from 0 to 0.2 - if (this.CurrentToken.Kind != SyntaxKind.DotToken || - !NoTriviaBetween(token2, this.CurrentToken)) - { - return dotDotToken; - } - - return this.AddError(dotDotToken, offset: dotDotToken.GetLeadingTriviaWidth(), length: 0, ErrorCode.ERR_TripleDotNotAllowed); + return SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); } private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) From f202cab90b72fbbcfc9a4c9bff1eb7329e27cdba Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 10:40:55 -0700 Subject: [PATCH 15/44] Update src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs --- src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs index f32350f84b5bf..930ced6a0a2bc 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs @@ -34,7 +34,7 @@ public override void Dispose() VerifyEnumeratorConsumed(); } - protected void VerifyEnumeratorConsumed() + private void VerifyEnumeratorConsumed() { if (_treeEnumerator != null) { From 853e317eff1302bae35c4274d69409683f4e8450 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 10:42:23 -0700 Subject: [PATCH 16/44] Simplify --- .../CSharp/Portable/Parser/LanguageParser.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 23816b3b15f25..9230fb13bfbb3 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11289,20 +11289,9 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) /// Check if we're currently at a .. sequence that can then be parsed out as a . public bool IsAtDotDotToken() - { - var token1 = this.CurrentToken; - if (token1.Kind != SyntaxKind.DotToken) - return false; - - var token2 = this.PeekToken(1); - if (token2.Kind != SyntaxKind.DotToken) - return false; - - return NoTriviaBetween(token1, token2)) - } - - private static bool IsNumericLiteralStartingWithDot(SyntaxToken token) - => token is { Kind: SyntaxKind.NumericLiteralToken, Text: ['.', >= '0' and <= '9', ..] }; + => this.CurrentToken is { Kind: SyntaxKind.DotToken } token1 && + this.PeekToken(1) is { Kind: SyntaxKind.DotToken } token2 && + NoTriviaBetween(token1, token2); /// Consume the next two tokens as a . Note: if three dot tokens /// are in a row, an error will be placed on the .. token to say that is illegal. From 82d2c50ec6efe41215d558e1858d91e153354f6d Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 11:02:16 -0700 Subject: [PATCH 17/44] Update quick scanner as well --- .../CSharp/Portable/Parser/LanguageParser.cs | 11 ++++++++++- src/Compilers/CSharp/Portable/Parser/Lexer.cs | 10 ++++++---- src/Compilers/CSharp/Portable/Parser/QuickScanner.cs | 5 +---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 9230fb13bfbb3..60614b0be9960 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11301,7 +11301,16 @@ public SyntaxToken EatDotDotToken() var token1 = this.EatToken(); var token2 = this.EatToken(); - return SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + var dotDotToken = SyntaxFactory.Token(token1.GetLeadingTrivia(), SyntaxKind.DotDotToken, token2.GetTrailingTrivia()); + if (this.CurrentToken is { Kind: SyntaxKind.DotToken } token3 && + NoTriviaBetween(token2, token3)) + { + // Give a specific error about `...` being illegal so we can reserve that syntax for the language for + // the future. + return AddError(dotDotToken, offset: dotDotToken.GetLeadingTriviaWidth(), length: 0, ErrorCode.ERR_TripleDotNotAllowed); + } + + return dotDotToken; } private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs index 62be0e60197f3..a34f9cdd9b3a3 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer.cs @@ -68,7 +68,7 @@ internal enum XmlDocCommentStyle Delimited = 1 } - internal partial class Lexer : AbstractLexer + internal sealed partial class Lexer : AbstractLexer { private const int TriviaListInitialCapacity = 8; @@ -465,10 +465,12 @@ private void ScanSyntaxToken(ref TokenInfo info) if (atDotPosition >= 1 && atDotPosition == this.TextWindow.LexemeStartPosition) { - // we have something like .0 this could be a fp number *except* the case where we have `..0` - // in that case, we want two dots followed by an integer (which will be treated as a range expression). + // We have something like: .0 // - // Move back one space to see what's before this dot. + // This could be a fp number *except* the case where we have `..0` in that case, we want two + // dots followed by an integer (which will be treated as a range expression). + // + // Move back one space to see what's before this dot and adjust accordingly. this.TextWindow.Reset(atDotPosition - 1); var priorCharacterIsDot = this.TextWindow.PeekChar() is '.'; diff --git a/src/Compilers/CSharp/Portable/Parser/QuickScanner.cs b/src/Compilers/CSharp/Portable/Parser/QuickScanner.cs index 13d159ced8b5d..0084d14b53f80 100644 --- a/src/Compilers/CSharp/Portable/Parser/QuickScanner.cs +++ b/src/Compilers/CSharp/Portable/Parser/QuickScanner.cs @@ -4,9 +4,6 @@ using System; using System.Diagnostics; -using Microsoft.CodeAnalysis.CSharp.Symbols; -using Microsoft.CodeAnalysis.CSharp.Syntax; -using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax @@ -152,7 +149,7 @@ private enum CharFlags : byte (byte)QuickScanState.FollowingCR, // CR (byte)QuickScanState.DoneAfterNext, // LF (byte)QuickScanState.Done, // Letter - (byte)QuickScanState.Number, // Digit + (byte)QuickScanState.Bad, // Dot followed by number. Could be a fp `.0` or could be a range + num `..0`. Can't tell here. (byte)QuickScanState.Done, // Punct (byte)QuickScanState.Bad, // Dot (DotDot range token, exit so that we handle it in subsequent scanning code) (byte)QuickScanState.Done, // Compound From 1cdc99b451192191bb17929723a0b2f5914a0648 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 11:08:21 -0700 Subject: [PATCH 18/44] More cases --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 60614b0be9960..1be9bfdc66ff5 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11682,7 +11682,9 @@ private bool CanStartConsequenceExpression() var nextTokenKind = this.PeekToken(1).Kind; // ?. is always the start of of a consequence expression. - if (nextTokenKind == SyntaxKind.DotToken) + // + // ?.. is a ternary with a range expression as it's 'whenTrue' clause. + if (nextTokenKind == SyntaxKind.DotToken && !IsAtDotDotToken()) return true; if (nextTokenKind == SyntaxKind.OpenBracketToken) From f5557ec3fab50dda2fa334aa1f572080b47883af Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 11:12:39 -0700 Subject: [PATCH 19/44] Consequence parsing --- .../CSharp/Portable/Parser/LanguageParser.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 1be9bfdc66ff5..bb2627b93a3f7 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11289,8 +11289,11 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) /// Check if we're currently at a .. sequence that can then be parsed out as a . public bool IsAtDotDotToken() - => this.CurrentToken is { Kind: SyntaxKind.DotToken } token1 && - this.PeekToken(1) is { Kind: SyntaxKind.DotToken } token2 && + => IsAtDotDotToken(this.CurrentToken, this.PeekToken(1)); + + public static bool IsAtDotDotToken(SyntaxToken token1, SyntaxToken token2) + => token1.Kind == SyntaxKind.DotToken && + token2.Kind == SyntaxKind.DotToken && NoTriviaBetween(token1, token2); /// Consume the next two tokens as a . Note: if three dot tokens @@ -11679,12 +11682,13 @@ private ExpressionSyntax ParsePostFixExpression(ExpressionSyntax expr) private bool CanStartConsequenceExpression() { Debug.Assert(this.CurrentToken.Kind == SyntaxKind.QuestionToken); - var nextTokenKind = this.PeekToken(1).Kind; + var nextToken = this.PeekToken(1); + var nextTokenKind = nextToken.Kind; - // ?. is always the start of of a consequence expression. + // ?. is always the start of of a consequence expression. // - // ?.. is a ternary with a range expression as it's 'whenTrue' clause. - if (nextTokenKind == SyntaxKind.DotToken && !IsAtDotDotToken()) + // ?.. is a ternary with a range expression as it's 'whenTrue' clause. + if (nextTokenKind == SyntaxKind.DotToken && !IsAtDotDotToken(nextToken, this.PeekToken(2))) return true; if (nextTokenKind == SyntaxKind.OpenBracketToken) From 46c2a9b656771bb65ad8ddbebc052c451f461098 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 11:26:42 -0700 Subject: [PATCH 20/44] Fixup test --- .../IncrementalParsing/IncrementalParsingTests.cs | 2 +- .../Test/Syntax/Parsing/ExpressionParsingTests.cs | 10 +++++----- .../Parsing/MemberDeclarationParsingTests.cs | 15 +++++---------- .../Parsing/PatternParsingTests_ListPatterns.cs | 10 +++++----- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs index 352274a2b09c7..0c6d1eb9a28a0 100644 --- a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs @@ -658,7 +658,7 @@ public void M2() Diagnostic(ErrorCode.ERR_InvalidMemberDecl, "=").WithArguments("=").WithLocation(15, 15), // (15,18): error CS1001: Identifier expected // y = [.. z]; - Diagnostic(ErrorCode.ERR_IdentifierExpected, "..").WithLocation(15, 18), + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(15, 18), // (15,19): error CS1001: Identifier expected // y = [.. z]; Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(15, 19), diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs index 35c676b2131ec..f6962b86b675e 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs @@ -5284,10 +5284,14 @@ public void IndexExpression() public void RangeExpression_ThreeDots() { UsingExpression("1...2", - // (1,2): error CS8401: Unexpected character sequence '...' + // (1,1): error CS1073: Unexpected token '.' + // 1...2 + Diagnostic(ErrorCode.ERR_UnexpectedToken, "1..").WithArguments(".").WithLocation(1, 1), + // (1,2): error CS8635: Unexpected character sequence '...' // 1...2 Diagnostic(ErrorCode.ERR_TripleDotNotAllowed, "").WithLocation(1, 2)); + N(SyntaxKind.RangeExpression); { N(SyntaxKind.NumericLiteralExpression); @@ -5295,10 +5299,6 @@ public void RangeExpression_ThreeDots() N(SyntaxKind.NumericLiteralToken, "1"); } N(SyntaxKind.DotDotToken); - N(SyntaxKind.NumericLiteralExpression); - { - N(SyntaxKind.NumericLiteralToken, ".2"); - } } EOF(); } diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/MemberDeclarationParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/MemberDeclarationParsingTests.cs index a98c94c918dd3..863aad136bf7c 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/MemberDeclarationParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/MemberDeclarationParsingTests.cs @@ -4118,8 +4118,7 @@ public void OperatorDeclaration_ExplicitImplementation_21() UsingDeclaration("public int I..operator +(int x) => x;", options: options.WithLanguageVersion(LanguageVersion.Preview), // (1,14): error CS1001: Identifier expected // public int I..operator +(int x) => x; - Diagnostic(ErrorCode.ERR_IdentifierExpected, ".operato").WithLocation(1, 14) - ); + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(1, 14)); N(SyntaxKind.OperatorDeclaration); { @@ -5964,8 +5963,7 @@ public void OperatorDeclaration_ExplicitImplementation_43() UsingDeclaration("int I..operator +(int x) => x;", options: options.WithLanguageVersion(LanguageVersion.Preview), // (1,7): error CS1001: Identifier expected // int I..operator +(int x) => x; - Diagnostic(ErrorCode.ERR_IdentifierExpected, ".operato").WithLocation(1, 7) - ); + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(1, 7)); N(SyntaxKind.OperatorDeclaration); { @@ -6088,8 +6086,7 @@ public void OperatorDeclaration_ExplicitImplementation_45() UsingDeclaration("int N.I..operator +(int x) => x;", options: options.WithLanguageVersion(LanguageVersion.Preview), // (1,9): error CS1001: Identifier expected // int N.I..operator +(int x) => x; - Diagnostic(ErrorCode.ERR_IdentifierExpected, ".operato").WithLocation(1, 9) - ); + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(1, 9)); N(SyntaxKind.OperatorDeclaration); { @@ -7819,8 +7816,7 @@ public void ConversionDeclaration_ExplicitImplementation_21() UsingDeclaration("explicit I..operator int(int x) => x;", options: options.WithLanguageVersion(LanguageVersion.Preview), // (1,12): error CS1001: Identifier expected // explicit I..operator int(int x) => x; - Diagnostic(ErrorCode.ERR_IdentifierExpected, ".operato").WithLocation(1, 12) - ); + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(1, 12)); N(SyntaxKind.ConversionOperatorDeclaration); { @@ -8551,8 +8547,7 @@ public void ConversionDeclaration_ExplicitImplementation_34() UsingDeclaration("explicit N.I..operator int(int x) => x;", options: options.WithLanguageVersion(LanguageVersion.Preview), // (1,14): error CS1001: Identifier expected // explicit N.I..operator int(int x) => x; - Diagnostic(ErrorCode.ERR_IdentifierExpected, ".operato").WithLocation(1, 14) - ); + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(1, 14)); N(SyntaxKind.ConversionOperatorDeclaration); { diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs index 422e1f3cb456f..51b7a67e413e6 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs @@ -767,9 +767,9 @@ public void SlicePattern_10() public void SlicePattern_11() { UsingExpression(@"c is [var x ..]", - // (1,13): error CS1003: Syntax error, ',' expected - // c is [var x ..] - Diagnostic(ErrorCode.ERR_SyntaxError, "..").WithArguments(",").WithLocation(1, 13)); + // (1,13): error CS1003: Syntax error, ',' expected + // c is [var x ..] + Diagnostic(ErrorCode.ERR_SyntaxError, ".").WithArguments(",").WithLocation(1, 13)); N(SyntaxKind.IsPatternExpression); { @@ -837,7 +837,7 @@ public void SlicePattern_13() UsingExpression(@"c is [[]..]", // (1,9): error CS1003: Syntax error, ',' expected // c is [[]..] - Diagnostic(ErrorCode.ERR_SyntaxError, "..").WithArguments(",").WithLocation(1, 9)); + Diagnostic(ErrorCode.ERR_SyntaxError, ".").WithArguments(",").WithLocation(1, 9)); N(SyntaxKind.IsPatternExpression); { @@ -871,7 +871,7 @@ public void SlicePattern_14() UsingExpression(@"c is not p ..", // (1,13): error CS1001: Identifier expected // c is not p .. - Diagnostic(ErrorCode.ERR_IdentifierExpected, "").WithLocation(1, 13), + Diagnostic(ErrorCode.ERR_IdentifierExpected, ".").WithLocation(1, 13), // (1,14): error CS1001: Identifier expected // c is not p .. Diagnostic(ErrorCode.ERR_IdentifierExpected, "").WithLocation(1, 14)); From 2b2ab956dded32ff2d6c9d656406000f89667a0e Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 12:01:58 -0700 Subject: [PATCH 21/44] More precise errors --- .../CSharp/Portable/Parser/LanguageParser.cs | 50 ++++++++++++------- .../CollectionExpressionParsingTests.cs | 17 ++----- .../Syntax/Parsing/ExpressionParsingTests.cs | 8 +-- .../PatternParsingTests_ListPatterns.cs | 3 +- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index bb2627b93a3f7..ae13603197f1e 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10985,7 +10985,7 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, // We either have a binary or assignment operator here, or we're finished. var tk = this.CurrentToken.ContextualKind; - bool isAssignmentOperator = false; + var isAssignmentOperator = false; SyntaxKind opKind; // If the set of expression continuations is updated here, please review ParseStatementAttributeDeclarations @@ -11025,24 +11025,19 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, { // check for >>, >>=, >>> or >>>= if (tk == SyntaxKind.GreaterThanToken - && this.PeekToken(1).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken - && NoTriviaBetween(this.CurrentToken, this.PeekToken(1))) // check to see if they really are adjacent + && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token1 + && NoTriviaBetween(this.CurrentToken, token1)) // check to see if they really are adjacent { - if (this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken) + if (token1.Kind == SyntaxKind.GreaterThanToken) { - if (this.PeekToken(2).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken - && NoTriviaBetween(this.PeekToken(1), this.PeekToken(2))) // check to see if they really are adjacent + if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 + && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent { - if (this.PeekToken(2).Kind == SyntaxKind.GreaterThanToken) - { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken); - } - else - { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); - isAssignmentOperator = true; - } + opKind = token2.Kind == SyntaxKind.GreaterThanToken + ? SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken) + : SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); + isAssignmentOperator = token2.Kind == SyntaxKind.GreaterThanEqualsToken; tokensToCombine = 3; } else @@ -11297,7 +11292,8 @@ public static bool IsAtDotDotToken(SyntaxToken token1, SyntaxToken token2) NoTriviaBetween(token1, token2); /// Consume the next two tokens as a . Note: if three dot tokens - /// are in a row, an error will be placed on the .. token to say that is illegal. + /// are in a row, an error will be placed on the .. token to say that is illegal, and single DotDot token + /// will be returned. public SyntaxToken EatDotDotToken() { Debug.Assert(IsAtDotDotToken()); @@ -11308,9 +11304,25 @@ public SyntaxToken EatDotDotToken() if (this.CurrentToken is { Kind: SyntaxKind.DotToken } token3 && NoTriviaBetween(token2, token3)) { - // Give a specific error about `...` being illegal so we can reserve that syntax for the language for - // the future. - return AddError(dotDotToken, offset: dotDotToken.GetLeadingTriviaWidth(), length: 0, ErrorCode.ERR_TripleDotNotAllowed); + // At least three dots directly in a row. Definitely mark that this is always illegal. We do not allow + // `...` at all in case we want to use that syntax in the future. + dotDotToken = AddError( + dotDotToken, + offset: dotDotToken.GetLeadingTriviaWidth(), + length: 0, + ErrorCode.ERR_TripleDotNotAllowed); + + // If we have exactly 3 dots in a row, then make the third dot into skipped trivia on the `..` as this + // is likely just a mistyped range/slice and we'll recover better if we don't try to process the 3rd dot + // as a member access or anything like that. + // + // If we have 4 dots in a row (`....`), then don't skip any of them. We'll let the caller handle the + // next two dots as a range/slice/whatever. + if (this.PeekToken(1) is not { Kind: SyntaxKind.DotToken } token4 || + !NoTriviaBetween(token3, token4)) + { + dotDotToken = AddSkippedSyntax(dotDotToken, this.EatToken(), trailing: true); + } } return dotDotToken; diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/CollectionExpressionParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/CollectionExpressionParsingTests.cs index 46b1fc99900ba..417f732a88673 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/CollectionExpressionParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/CollectionExpressionParsingTests.cs @@ -6992,10 +6992,7 @@ public void TestError5() UsingExpression("[...e]", // (1,2): error CS8635: Unexpected character sequence '...' // [...e] - Diagnostic(ErrorCode.ERR_TripleDotNotAllowed, "").WithLocation(1, 2), - // (1,4): error CS1525: Invalid expression term '.' - // [...e] - Diagnostic(ErrorCode.ERR_InvalidExprTerm, ".").WithArguments(".").WithLocation(1, 4)); + Diagnostic(ErrorCode.ERR_TripleDotNotAllowed, "").WithLocation(1, 2)); N(SyntaxKind.CollectionExpression); { @@ -7003,17 +7000,9 @@ public void TestError5() N(SyntaxKind.SpreadElement); { N(SyntaxKind.DotDotToken); - N(SyntaxKind.SimpleMemberAccessExpression); + N(SyntaxKind.IdentifierName); { - M(SyntaxKind.IdentifierName); - { - M(SyntaxKind.IdentifierToken); - } - N(SyntaxKind.DotToken); - N(SyntaxKind.IdentifierName); - { - N(SyntaxKind.IdentifierToken, "e"); - } + N(SyntaxKind.IdentifierToken, "e"); } } N(SyntaxKind.CloseBracketToken); diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs index f6962b86b675e..268d4e0ce5d5d 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/ExpressionParsingTests.cs @@ -5284,14 +5284,10 @@ public void IndexExpression() public void RangeExpression_ThreeDots() { UsingExpression("1...2", - // (1,1): error CS1073: Unexpected token '.' - // 1...2 - Diagnostic(ErrorCode.ERR_UnexpectedToken, "1..").WithArguments(".").WithLocation(1, 1), // (1,2): error CS8635: Unexpected character sequence '...' // 1...2 Diagnostic(ErrorCode.ERR_TripleDotNotAllowed, "").WithLocation(1, 2)); - N(SyntaxKind.RangeExpression); { N(SyntaxKind.NumericLiteralExpression); @@ -5299,6 +5295,10 @@ public void RangeExpression_ThreeDots() N(SyntaxKind.NumericLiteralToken, "1"); } N(SyntaxKind.DotDotToken); + N(SyntaxKind.NumericLiteralExpression); + { + N(SyntaxKind.NumericLiteralToken, "2"); + } } EOF(); } diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs index 51b7a67e413e6..e3c5ee9b98ed6 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/PatternParsingTests_ListPatterns.cs @@ -530,8 +530,7 @@ public void SlicePattern_04() UsingExpression(@"c is ....", // (1,6): error CS8635: Unexpected character sequence '...' // c is .... - Diagnostic(ErrorCode.ERR_TripleDotNotAllowed, "").WithLocation(1, 6) - ); + Diagnostic(ErrorCode.ERR_TripleDotNotAllowed, "").WithLocation(1, 6)); N(SyntaxKind.IsPatternExpression); { From 28acd3b6893a5972c3fe4e628319a6973c55c6f2 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 12:17:27 -0700 Subject: [PATCH 22/44] Specialized error message --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index ae13603197f1e..091616d9f6df1 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -3840,9 +3840,10 @@ private MemberDeclarationSyntax ParseOperatorDeclaration( } else { - //Consume whatever follows the operator keyword as the operator token. If it is not - //we'll add an error below (when we can guess the arity). - opToken = EatToken(); + // Consume whatever follows the operator keyword as the operator token. If it is not we'll add an + // error below (when we can guess the arity). Handle .. as well so we can give the user a good + // message if they do `operator ..` + opToken = IsAtDotDotToken() ? EatDotDotToken() : EatToken(); Debug.Assert(!opToken.IsMissing); opTokenErrorOffset = opToken.GetLeadingTriviaWidth(); opTokenErrorWidth = opToken.Width; From e57e96788ef99dcbbe555080048a7138d04e2ce4 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 13:29:22 -0700 Subject: [PATCH 23/44] Simplify expression parsing --- .../CSharp/Portable/Parser/LanguageParser.cs | 111 ++++++++---------- 1 file changed, 50 insertions(+), 61 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index f3db0e485c25d..99df6186f88f8 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10937,9 +10937,6 @@ private ExpressionSyntax ParseSubExpression(Precedence precedence) private ExpressionSyntax ParseSubExpressionCore(Precedence precedence) { - ExpressionSyntax leftOperand; - Precedence newPrecedence = 0; - // all of these are tokens that start statements and are invalid // to start a expression with. if we see one, then we must have // something like: @@ -10949,75 +10946,67 @@ private ExpressionSyntax ParseSubExpressionCore(Precedence precedence) // parse out a missing name node for the expression, and keep on going var tk = this.CurrentToken.Kind; if (IsInvalidSubExpression(tk)) - { return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); - } - // Parse a left operand -- possibly preceded by a unary operator. - if (IsExpectedPrefixUnaryOperator(tk)) - { - var opKind = SyntaxFacts.GetPrefixUnaryExpression(tk); - newPrecedence = GetPrecedence(opKind); - var opToken = this.EatToken(); - var operand = this.ParseSubExpression(newPrecedence); - leftOperand = _syntaxFactory.PrefixUnaryExpression(opKind, opToken, operand); - } - else if (tk == SyntaxKind.DotDotToken) + return ParseExpressionContinued(parseLeftOperand(), precedence); + + ExpressionSyntax parseLeftOperand() { - // Operator ".." here can either be a prefix unary operator or a stand alone empty range: - var opToken = this.EatToken(); - newPrecedence = GetPrecedence(SyntaxKind.RangeExpression); + // Parse a left operand -- possibly preceded by a unary operator. + if (IsExpectedPrefixUnaryOperator(tk)) + { + var opKind = SyntaxFacts.GetPrefixUnaryExpression(tk); + return _syntaxFactory.PrefixUnaryExpression( + opKind, + this.EatToken(), + this.ParseSubExpression(GetPrecedence(opKind))); + } - ExpressionSyntax rightOperand; - if (CanStartExpression()) + if (tk == SyntaxKind.DotDotToken) { - rightOperand = this.ParseSubExpression(newPrecedence); + return _syntaxFactory.RangeExpression( + leftOperand: null, + this.EatToken(), + CanStartExpression() + ? this.ParseSubExpression(GetPrecedence(SyntaxKind.RangeExpression)) + : null); } - else + + if (IsAwaitExpression()) { - rightOperand = null; + return _syntaxFactory.AwaitExpression( + this.EatContextualToken(SyntaxKind.AwaitKeyword), + this.ParseSubExpression(GetPrecedence(SyntaxKind.AwaitExpression))); } - leftOperand = _syntaxFactory.RangeExpression(leftOperand: null, opToken, rightOperand); - } - else if (IsAwaitExpression()) - { - newPrecedence = GetPrecedence(SyntaxKind.AwaitExpression); - leftOperand = _syntaxFactory.AwaitExpression( - this.EatContextualToken(SyntaxKind.AwaitKeyword), - this.ParseSubExpression(newPrecedence)); - } - else if (this.IsQueryExpression(mayBeVariableDeclaration: false, mayBeMemberDeclaration: false)) - { - leftOperand = this.ParseQueryExpression(precedence); - } - else if (this.CurrentToken.ContextualKind == SyntaxKind.FromKeyword && IsInQuery) - { - // If this "from" token wasn't the start of a query then it's not really an expression. - // Consume it so that we don't try to parse it again as the next argument in an - // argument list. - SyntaxToken skipped = this.EatToken(); // consume but skip "from" - skipped = this.AddError(skipped, ErrorCode.ERR_InvalidExprTerm, this.CurrentToken.Text); - leftOperand = AddTrailingSkippedSyntax(this.CreateMissingIdentifierName(), skipped); - } - else if (tk == SyntaxKind.ThrowKeyword) - { - var result = ParseThrowExpression(); - // we parse a throw expression even at the wrong precedence for better recovery - return (precedence <= Precedence.Coalescing) ? result : - this.AddError(result, ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); - } - else if (this.IsPossibleDeconstructionLeft(precedence)) - { - leftOperand = ParseDeclarationExpression(ParseTypeMode.Normal, isScoped: false); - } - else - { + if (this.IsQueryExpression(mayBeVariableDeclaration: false, mayBeMemberDeclaration: false)) + return this.ParseQueryExpression(precedence); + + if (this.CurrentToken.ContextualKind == SyntaxKind.FromKeyword && IsInQuery) + { + // If this "from" token wasn't the start of a query then it's not really an expression. + // Consume it so that we don't try to parse it again as the next argument in an + // argument list. + return AddTrailingSkippedSyntax( + this.CreateMissingIdentifierName(), + this.AddError(this.EatToken(), ErrorCode.ERR_InvalidExprTerm, this.CurrentToken.Text)); + } + + if (tk == SyntaxKind.ThrowKeyword) + { + var result = ParseThrowExpression(); + // we parse a throw expression even at the wrong precedence for better recovery + return precedence <= Precedence.Coalescing + ? result + : this.AddError(result, ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); + } + + if (this.IsPossibleDeconstructionLeft(precedence)) + return ParseDeclarationExpression(ParseTypeMode.Normal, isScoped: false); + // Not a unary operator - get a primary expression. - leftOperand = this.ParseTerm(precedence); + return this.ParseTerm(precedence); } - - return ParseExpressionContinued(leftOperand, precedence); } private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, Precedence precedence) From e6eb643469cdec7e746a88faaf5dfb2e3dc37600 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 13:52:03 -0700 Subject: [PATCH 24/44] Extract helper --- .../CSharp/Portable/Parser/LanguageParser.cs | 325 ++++++++++-------- 1 file changed, 173 insertions(+), 152 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 99df6186f88f8..f84af7172fb4e 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10948,9 +10948,9 @@ private ExpressionSyntax ParseSubExpressionCore(Precedence precedence) if (IsInvalidSubExpression(tk)) return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); - return ParseExpressionContinued(parseLeftOperand(), precedence); + return ParseExpressionContinued(parseLeftOperand(precedence), precedence); - ExpressionSyntax parseLeftOperand() + ExpressionSyntax parseLeftOperand(Precedence precedence) { // Parse a left operand -- possibly preceded by a unary operator. if (IsExpectedPrefixUnaryOperator(tk)) @@ -11011,12 +11011,32 @@ ExpressionSyntax parseLeftOperand() private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, Precedence precedence) { - while (true) + // Keep on expanding the left operand as long as what we see fits the precedence we're under. + while (tryExpandLeftOperand(ref leftOperand, precedence)) + continue; + + // Finally, consume a conditional expression if precedence allows it. + + // https://github.com/dotnet/csharpstandard/blob/standard-v6/standard/expressions.md#1115-conditional-operator: + // + // conditional_expression + // : null_coalescing_expression + // | null_coalescing_expression '?' expression ':' expression + // ; + // + // 1. Only take the conditional part of the expression if we're at or below its precedence. + // 2. When parsing the branches of the expression, parse at the highest precedence again ('expression'). + // This allows for things like assignments/lambdas in the branches of the conditional. + if (CurrentToken.Kind != SyntaxKind.QuestionToken || precedence > Precedence.Conditional) + return leftOperand; + + return consumeConditionalExpression(leftOperand); + + bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedence) { // We either have a binary or assignment operator here, or we're finished. var tk = this.CurrentToken.ContextualKind; - bool isAssignmentOperator = false; SyntaxKind opKind; // If the set of expression continuations is updated here, please review ParseStatementAttributeDeclarations @@ -11030,7 +11050,6 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, else if (IsExpectedAssignmentOperator(tk)) { opKind = SyntaxFacts.GetAssignmentExpression(tk); - isAssignmentOperator = true; } else if (tk == SyntaxKind.DotDotToken) { @@ -11046,64 +11065,93 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, } else { - break; + // Something that doesn't expand the current expression we're looking at. Bail out and see if we + // can end with a conditional expression. + return false; } - var newPrecedence = GetPrecedence(opKind); - // check for >>, >>=, >>> or >>>= - int tokensToCombine = 1; - if (tk == SyntaxKind.GreaterThanToken - && this.PeekToken(1).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken - && NoTriviaBetween(this.CurrentToken, this.PeekToken(1))) // check to see if they really are adjacent + var tokensToCombine = 1; { - if (this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken) + if (tk == SyntaxKind.GreaterThanToken + && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token1 + && NoTriviaBetween(this.CurrentToken, token1)) // check to see if they really are adjacent { - if (this.PeekToken(2).Kind is SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken - && NoTriviaBetween(this.PeekToken(1), this.PeekToken(2))) // check to see if they really are adjacent + if (token1.Kind == SyntaxKind.GreaterThanToken) { - if (this.PeekToken(2).Kind == SyntaxKind.GreaterThanToken) + if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 + && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken); + if (token2.Kind == SyntaxKind.GreaterThanToken) + { + opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken); + } + else + { + opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); + } + + tokensToCombine = 3; } else { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); - isAssignmentOperator = true; + opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); + tokensToCombine = 2; } - - tokensToCombine = 3; } else { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); + opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); tokensToCombine = 2; } } - else - { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); - isAssignmentOperator = true; - tokensToCombine = 2; - } - - newPrecedence = GetPrecedence(opKind); } - // Check the precedence to see if we should "take" this operator + var newPrecedence = GetPrecedence(opKind); + + // Check the precedence to see if we should "take" this operator. A lower precedence means what's + // coming isn't a child of us, but rather we will be a child of it. So we bail out and let any higher + // up expression parsing consume it with us as the left side. if (newPrecedence < precedence) - { - break; - } + return false; // Same precedence, but not right-associative -- deal with this "later" if ((newPrecedence == precedence) && !IsRightAssociative(opKind)) + return false; + + // We'll "take" this operator, as precedence is tentatively OK. + + // Combine tokens into a single token if needed + SyntaxToken operatorToken; + if (tokensToCombine == 1) { - break; + operatorToken = this.EatContextualToken(tk); + } + else if (tokensToCombine == 2) + { + var token1 = this.EatContextualToken(tk); + var token2 = this.EatToken(); + operatorToken = SyntaxFactory.Token( + token1.GetLeadingTrivia(), + token2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken, + token2.GetTrailingTrivia()); } + else if (tokensToCombine == 3) + { + var token1 = this.EatContextualToken(tk); + var token2 = this.EatToken(); + var token3 = this.EatToken(); - // We'll "take" this operator, as precedence is tentatively OK. - var opToken = this.EatContextualToken(tk); + Debug.Assert(token2.Kind == SyntaxKind.GreaterThanToken); + operatorToken = SyntaxFactory.Token( + token1.GetLeadingTrivia(), + token3.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, + token3.GetTrailingTrivia()); + } + else + { + throw ExceptionUtilities.UnexpectedValue(tokensToCombine); + } var leftPrecedence = GetPrecedence(leftOperand.Kind); if (newPrecedence > leftPrecedence) @@ -11119,40 +11167,26 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, // with an anonymous method expression or a lambda expression with a block body. No // further parsing will find a way to fix things up, so we accept the operator but issue // a diagnostic. - ErrorCode errorCode = leftOperand.Kind == SyntaxKind.IsPatternExpression ? ErrorCode.ERR_UnexpectedToken : ErrorCode.WRN_PrecedenceInversion; - opToken = this.AddError(opToken, errorCode, opToken.Text); - } - - // Combine tokens into a single token if needed - if (tokensToCombine == 2) - { - var opToken2 = this.EatToken(); - var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken; - opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); - } - else if (tokensToCombine == 3) - { - var opToken2 = this.EatToken(); - Debug.Assert(opToken2.Kind == SyntaxKind.GreaterThanToken); - opToken2 = this.EatToken(); - var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; - opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); - } - else if (tokensToCombine != 1) - { - throw ExceptionUtilities.UnexpectedValue(tokensToCombine); + operatorToken = this.AddError( + operatorToken, + leftOperand.Kind == SyntaxKind.IsPatternExpression ? ErrorCode.ERR_UnexpectedToken : ErrorCode.WRN_PrecedenceInversion, + operatorToken.Text); } if (opKind == SyntaxKind.AsExpression) { - var type = this.ParseType(ParseTypeMode.AsExpression); - leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, opToken, type); + leftOperand = _syntaxFactory.BinaryExpression( + opKind, leftOperand, operatorToken, this.ParseType(ParseTypeMode.AsExpression)); + return true; } - else if (opKind == SyntaxKind.IsExpression) + + if (opKind == SyntaxKind.IsExpression) { - leftOperand = ParseIsExpression(leftOperand, opToken); + leftOperand = ParseIsExpression(leftOperand, operatorToken); + return true; } - else if (isAssignmentOperator) + + if (SyntaxFacts.IsAssignmentExpression(opKind)) { ExpressionSyntax rhs; @@ -11169,117 +11203,104 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, rhs = this.ParseSubExpression(newPrecedence); } - leftOperand = _syntaxFactory.AssignmentExpression(opKind, leftOperand, opToken, rhs); + leftOperand = _syntaxFactory.AssignmentExpression(opKind, leftOperand, operatorToken, rhs); + return true; } - else if (opKind == SyntaxKind.SwitchExpression) + + if (opKind == SyntaxKind.SwitchExpression) { - leftOperand = ParseSwitchExpression(leftOperand, opToken); + leftOperand = ParseSwitchExpression(leftOperand, operatorToken); + return true; } - else if (opKind == SyntaxKind.WithExpression) + + if (opKind == SyntaxKind.WithExpression) { - leftOperand = ParseWithExpression(leftOperand, opToken); + leftOperand = ParseWithExpression(leftOperand, operatorToken); + return true; } - else if (tk == SyntaxKind.DotDotToken) - { - // Operator ".." here can either be a binary or a postfix unary operator: - Debug.Assert(opKind == SyntaxKind.RangeExpression); - ExpressionSyntax rightOperand; - if (CanStartExpression()) - { - newPrecedence = GetPrecedence(opKind); - rightOperand = this.ParseSubExpression(newPrecedence); - } - else - { - rightOperand = null; - } - - leftOperand = _syntaxFactory.RangeExpression(leftOperand, opToken, rightOperand); - } - else + if (opKind == SyntaxKind.RangeExpression) { - Debug.Assert(IsExpectedBinaryOperator(tk)); - leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, opToken, this.ParseSubExpression(newPrecedence)); + leftOperand = _syntaxFactory.RangeExpression( + leftOperand, + operatorToken, + CanStartExpression() + ? this.ParseSubExpression(Precedence.Range) + : null); + return true; } + + Debug.Assert(IsExpectedBinaryOperator(tk)); + leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); + return true; } - // https://github.com/dotnet/csharpstandard/blob/standard-v6/standard/expressions.md#1115-conditional-operator: - // - // conditional_expression - // : null_coalescing_expression - // | null_coalescing_expression '?' expression ':' expression - // ; - // - // 1. Only take the conditional part of the expression if we're at or below its precedence. - // 2. When parsing the branches of the expression, parse at the highest precedence again ('expression'). - // This allows for things like assignments/lambdas in the branches of the conditional. - if (CurrentToken.Kind != SyntaxKind.QuestionToken || precedence > Precedence.Conditional) - return leftOperand; + ConditionalExpressionSyntax consumeConditionalExpression(ExpressionSyntax leftOperand) + { + // Complex ambiguity with `?` and collection-expressions. Specifically: b?[c]:d + // + // On its own, we want that to be a conditional expression with a collection expression in it. However, for + // back compat, we need to make sure that `a ? b?[c] : d` sees the inner `b?[c]` as a + // conditional-access-expression. So, if after consuming the portion after the initial `?` if we do not + // have the `:` we need, and we can see a `?[` in that portion of the parse, then we retry consuming the + // when-true portion, but this time forcing the prior way of handling `?[`. + var questionToken = this.EatToken(); - // Complex ambiguity with `?` and collection-expressions. Specifically: b?[c]:d - // - // On its own, we want that to be a conditional expression with a collection expression in it. However, for - // back compat, we need to make sure that `a ? b?[c] : d` sees the inner `b?[c]` as a - // conditional-access-expression. So, if after consuming the portion after the initial `?` if we do not - // have the `:` we need, and we can see a `?[` in that portion of the parse, then we retry consuming the - // when-true portion, but this time forcing the prior way of handling `?[`. - var questionToken = this.EatToken(); + using var afterQuestionToken = this.GetDisposableResetPoint(resetOnDispose: false); + var whenTrue = this.ParsePossibleRefExpression(); - using var afterQuestionToken = this.GetDisposableResetPoint(resetOnDispose: false); - var whenTrue = this.ParsePossibleRefExpression(); + if (this.CurrentToken.Kind != SyntaxKind.ColonToken && + !this.ForceConditionalAccessExpression && + containsTernaryCollectionToReinterpret(whenTrue)) + { + // Keep track of where we are right now in case the new parse doesn't make things better. + using var originalAfterWhenTrue = this.GetDisposableResetPoint(resetOnDispose: false); - if (this.CurrentToken.Kind != SyntaxKind.ColonToken && - !this.ForceConditionalAccessExpression && - containsTernaryCollectionToReinterpret(whenTrue)) - { - // Keep track of where we are right now in case the new parse doesn't make things better. - using var originalAfterWhenTrue = this.GetDisposableResetPoint(resetOnDispose: false); + // Go back to right after the `?` + afterQuestionToken.Reset(); - // Go back to right after the `?` - afterQuestionToken.Reset(); + // try reparsing with `?[` as a conditional access, not a ternary+collection + this.ForceConditionalAccessExpression = true; + var newWhenTrue = this.ParsePossibleRefExpression(); + this.ForceConditionalAccessExpression = false; - // try reparsing with `?[` as a conditional access, not a ternary+collection - this.ForceConditionalAccessExpression = true; - var newWhenTrue = this.ParsePossibleRefExpression(); - this.ForceConditionalAccessExpression = false; + if (this.CurrentToken.Kind == SyntaxKind.ColonToken) + { + // if we now are at a colon, this was preferred parse. + whenTrue = newWhenTrue; + } + else + { + // retrying the parse didn't help. Use the original interpretation. + originalAfterWhenTrue.Reset(); + } + } - if (this.CurrentToken.Kind == SyntaxKind.ColonToken) + if (this.CurrentToken.Kind == SyntaxKind.EndOfFileToken && this.lexer.InterpolationFollowedByColon) { - // if we now are at a colon, this was preferred parse. - whenTrue = newWhenTrue; + // We have an interpolated string with an interpolation that contains a conditional expression. + // Unfortunately, the precedence demands that the colon is considered to signal the start of the + // format string. Without this code, the compiler would complain about a missing colon, and point + // to the colon that is present, which would be confusing. We aim to give a better error message. + var conditionalExpression = _syntaxFactory.ConditionalExpression( + leftOperand, + questionToken, + whenTrue, + SyntaxFactory.MissingToken(SyntaxKind.ColonToken), + _syntaxFactory.IdentifierName(SyntaxFactory.MissingToken(SyntaxKind.IdentifierToken))); + return this.AddError(conditionalExpression, ErrorCode.ERR_ConditionalInInterpolation); } else { - // retrying the parse didn't help. Use the original interpretation. - originalAfterWhenTrue.Reset(); + return _syntaxFactory.ConditionalExpression( + leftOperand, + questionToken, + whenTrue, + this.EatToken(SyntaxKind.ColonToken), + this.ParsePossibleRefExpression()); } } - if (this.CurrentToken.Kind == SyntaxKind.EndOfFileToken && this.lexer.InterpolationFollowedByColon) - { - // We have an interpolated string with an interpolation that contains a conditional expression. - // Unfortunately, the precedence demands that the colon is considered to signal the start of the - // format string. Without this code, the compiler would complain about a missing colon, and point - // to the colon that is present, which would be confusing. We aim to give a better error message. - leftOperand = _syntaxFactory.ConditionalExpression( - leftOperand, - questionToken, - whenTrue, - SyntaxFactory.MissingToken(SyntaxKind.ColonToken), - _syntaxFactory.IdentifierName(SyntaxFactory.MissingToken(SyntaxKind.IdentifierToken))); - return this.AddError(leftOperand, ErrorCode.ERR_ConditionalInInterpolation); - } - else - { - return _syntaxFactory.ConditionalExpression( - leftOperand, - questionToken, - whenTrue, - this.EatToken(SyntaxKind.ColonToken), - this.ParsePossibleRefExpression()); - } - static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) { var stack = ArrayBuilder.GetInstance(); From 4f5e6fc41120d771a98ade5d51fb99165b7ec2ab Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 13:54:00 -0700 Subject: [PATCH 25/44] simplify --- .../CSharp/Portable/Parser/LanguageParser.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index f84af7172fb4e..eb3d5b47a3c20 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11082,14 +11082,9 @@ bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedenc if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent { - if (token2.Kind == SyntaxKind.GreaterThanToken) - { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken); - } - else - { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); - } + opKind = token2.Kind == SyntaxKind.GreaterThanToken + ? SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken) + : SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); tokensToCombine = 3; } From b8c14af1b0f771bec1326036b84e0c1480024a1e Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 14:16:39 -0700 Subject: [PATCH 26/44] simplify --- .../CSharp/Portable/Parser/LanguageParser.cs | 203 +++++++++--------- 1 file changed, 100 insertions(+), 103 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index eb3d5b47a3c20..3397ef56583a8 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10948,9 +10948,10 @@ private ExpressionSyntax ParseSubExpressionCore(Precedence precedence) if (IsInvalidSubExpression(tk)) return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); - return ParseExpressionContinued(parseLeftOperand(precedence), precedence); + return ParseExpressionContinued(parseUnaryOrPrimaryExpression(precedence), precedence); - ExpressionSyntax parseLeftOperand(Precedence precedence) + // Parses out a unary expression, or something with higher precedence (cast, addressof, primary). + ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) { // Parse a left operand -- possibly preceded by a unary operator. if (IsExpectedPrefixUnaryOperator(tk)) @@ -11005,14 +11006,20 @@ ExpressionSyntax parseLeftOperand(Precedence precedence) return ParseDeclarationExpression(ParseTypeMode.Normal, isScoped: false); // Not a unary operator - get a primary expression. - return this.ParseTerm(precedence); + return this.ParsePrimaryExpression(precedence); } } - private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, Precedence precedence) + /// + /// Takes in an initial unary expression or primary expression, and then consumes what follows as long as as + /// it's precedence is either lower than the we're parsing currently (or equal, to + /// that precedence if we have something right-associative . + /// + private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimaryExpression, Precedence precedence) { + var expandedExpression = unaryOrPrimaryExpression; // Keep on expanding the left operand as long as what we see fits the precedence we're under. - while (tryExpandLeftOperand(ref leftOperand, precedence)) + while (tryExpandExpression(ref expandedExpression, precedence)) continue; // Finally, consume a conditional expression if precedence allows it. @@ -11027,14 +11034,16 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax leftOperand, // 1. Only take the conditional part of the expression if we're at or below its precedence. // 2. When parsing the branches of the expression, parse at the highest precedence again ('expression'). // This allows for things like assignments/lambdas in the branches of the conditional. - if (CurrentToken.Kind != SyntaxKind.QuestionToken || precedence > Precedence.Conditional) - return leftOperand; + if (this.CurrentToken.Kind == SyntaxKind.QuestionToken && precedence <= Precedence.Conditional) + return consumeConditionalExpression(expandedExpression); - return consumeConditionalExpression(leftOperand); + return expandedExpression; - bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedence) + bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence) { - // We either have a binary or assignment operator here, or we're finished. + // Look for operators that can follow what we've seen so far, and which are acceptable at this + // precedence level. Examples include binary operator, assignment operators, range operators `..`, as + // well as `switch` and `with` clauses. var tk = this.CurrentToken.ContextualKind; SyntaxKind opKind; @@ -11043,7 +11052,31 @@ bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedenc // to see if it may need a similar look-ahead check to determine if something is a collection expression versus // an attribute. - if (IsExpectedBinaryOperator(tk)) + // check for >>, >>=, >>> or >>>= + if (tk == SyntaxKind.GreaterThanToken + && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token1 + && NoTriviaBetween(this.CurrentToken, token1)) // check to see if they really are adjacent + { + if (token1.Kind == SyntaxKind.GreaterThanToken) + { + if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 + && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent + { + opKind = token2.Kind == SyntaxKind.GreaterThanToken + ? SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken) + : SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); + } + else + { + opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); + } + } + else + { + opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); + } + } + else if (IsExpectedBinaryOperator(tk)) { opKind = SyntaxFacts.GetBinaryExpression(tk); } @@ -11070,38 +11103,6 @@ bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedenc return false; } - // check for >>, >>=, >>> or >>>= - var tokensToCombine = 1; - { - if (tk == SyntaxKind.GreaterThanToken - && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token1 - && NoTriviaBetween(this.CurrentToken, token1)) // check to see if they really are adjacent - { - if (token1.Kind == SyntaxKind.GreaterThanToken) - { - if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 - && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent - { - opKind = token2.Kind == SyntaxKind.GreaterThanToken - ? SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken) - : SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); - - tokensToCombine = 3; - } - else - { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); - tokensToCombine = 2; - } - } - else - { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); - tokensToCombine = 2; - } - } - } - var newPrecedence = GetPrecedence(opKind); // Check the precedence to see if we should "take" this operator. A lower precedence means what's @@ -11116,40 +11117,9 @@ bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedenc // We'll "take" this operator, as precedence is tentatively OK. - // Combine tokens into a single token if needed - SyntaxToken operatorToken; - if (tokensToCombine == 1) - { - operatorToken = this.EatContextualToken(tk); - } - else if (tokensToCombine == 2) - { - var token1 = this.EatContextualToken(tk); - var token2 = this.EatToken(); - operatorToken = SyntaxFactory.Token( - token1.GetLeadingTrivia(), - token2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken, - token2.GetTrailingTrivia()); - } - else if (tokensToCombine == 3) - { - var token1 = this.EatContextualToken(tk); - var token2 = this.EatToken(); - var token3 = this.EatToken(); - - Debug.Assert(token2.Kind == SyntaxKind.GreaterThanToken); - operatorToken = SyntaxFactory.Token( - token1.GetLeadingTrivia(), - token3.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, - token3.GetTrailingTrivia()); - } - else - { - throw ExceptionUtilities.UnexpectedValue(tokensToCombine); - } + var operatorToken = eatOperatorToken(opKind); - var leftPrecedence = GetPrecedence(leftOperand.Kind); - if (newPrecedence > leftPrecedence) + if (newPrecedence > GetPrecedence(leftOperand.Kind)) { // Normally, a left operand with a looser precedence will consume all right operands that // have a tighter precedence. For example, in the expression `a + b * c`, the `* c` part @@ -11172,16 +11142,29 @@ bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedenc { leftOperand = _syntaxFactory.BinaryExpression( opKind, leftOperand, operatorToken, this.ParseType(ParseTypeMode.AsExpression)); - return true; } - - if (opKind == SyntaxKind.IsExpression) + else if (opKind == SyntaxKind.IsExpression) { leftOperand = ParseIsExpression(leftOperand, operatorToken); - return true; } - - if (SyntaxFacts.IsAssignmentExpression(opKind)) + else if (opKind == SyntaxKind.SwitchExpression) + { + leftOperand = ParseSwitchExpression(leftOperand, operatorToken); + } + else if (opKind == SyntaxKind.WithExpression) + { + leftOperand = ParseWithExpression(leftOperand, operatorToken); + } + else if (opKind == SyntaxKind.RangeExpression) + { + leftOperand = _syntaxFactory.RangeExpression( + leftOperand, + operatorToken, + CanStartExpression() + ? this.ParseSubExpression(Precedence.Range) + : null); + } + else if (SyntaxFacts.IsAssignmentExpression(opKind)) { ExpressionSyntax rhs; @@ -11199,35 +11182,49 @@ bool tryExpandLeftOperand(ref ExpressionSyntax leftOperand, Precedence precedenc } leftOperand = _syntaxFactory.AssignmentExpression(opKind, leftOperand, operatorToken, rhs); - return true; } - - if (opKind == SyntaxKind.SwitchExpression) + else if (IsExpectedBinaryOperator(tk)) { - leftOperand = ParseSwitchExpression(leftOperand, operatorToken); - return true; + leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); } - - if (opKind == SyntaxKind.WithExpression) + else { - leftOperand = ParseWithExpression(leftOperand, operatorToken); - return true; + throw ExceptionUtilities.Unreachable(); } - if (opKind == SyntaxKind.RangeExpression) + return true; + } + + SyntaxToken eatOperatorToken(SyntaxKind operatorKind) + { + // Combine tokens into a single token if needed + if (operatorKind is SyntaxKind.RightShiftExpression or SyntaxKind.RightShiftAssignmentExpression) { - leftOperand = _syntaxFactory.RangeExpression( - leftOperand, - operatorToken, - CanStartExpression() - ? this.ParseSubExpression(Precedence.Range) - : null); - return true; + // >> or >>= + return SyntaxFactory.Token( + this.EatToken().GetLeadingTrivia(), + operatorKind == SyntaxKind.RightShiftExpression ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken, + this.EatToken().GetTrailingTrivia()); + } + else if (operatorKind is SyntaxKind.UnsignedRightShiftExpression or SyntaxKind.UnsignedRightShiftAssignmentExpression) + { + // >>> or >>>= + var leadingTrivia = this.EatToken().GetLeadingTrivia(); - Debug.Assert(IsExpectedBinaryOperator(tk)); - leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); - return true; + // Skip middle token. + _ = this.EatToken(); + + return SyntaxFactory.Token( + leadingTrivia, + operatorKind == SyntaxKind.UnsignedRightShiftExpression ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, + this.EatToken().GetTrailingTrivia()); + } + else + { + // Normal operator + return this.EatContextualToken(this.CurrentToken.ContextualKind); + } } ConditionalExpressionSyntax consumeConditionalExpression(ExpressionSyntax leftOperand) @@ -11354,7 +11351,7 @@ private ExpressionSyntax ParseIsExpression(ExpressionSyntax leftOperand, SyntaxT }; } - private ExpressionSyntax ParseTerm(Precedence precedence) + private ExpressionSyntax ParsePrimaryExpression(Precedence precedence) => this.ParsePostFixExpression(ParseTermWithoutPostfix(precedence)); private ExpressionSyntax ParseTermWithoutPostfix(Precedence precedence) From cc0efdbabc40734d4df7dc7429607071d4286698 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 14:25:45 -0700 Subject: [PATCH 27/44] simplify --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 3397ef56583a8..b3bd17de5e7bf 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10969,7 +10969,7 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) leftOperand: null, this.EatToken(), CanStartExpression() - ? this.ParseSubExpression(GetPrecedence(SyntaxKind.RangeExpression)) + ? this.ParseSubExpression(Precedence.Range) : null); } @@ -11018,6 +11018,7 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimaryExpression, Precedence precedence) { var expandedExpression = unaryOrPrimaryExpression; + // Keep on expanding the left operand as long as what we see fits the precedence we're under. while (tryExpandExpression(ref expandedExpression, precedence)) continue; From 28fad182ab50bb8fa8bcf17039ff0b67da1c5c72 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 17 Oct 2024 14:40:03 -0700 Subject: [PATCH 28/44] inline --- .../CSharp/Portable/Parser/LanguageParser.cs | 458 +++++++++--------- 1 file changed, 234 insertions(+), 224 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index b3bd17de5e7bf..40246f5d3a3e3 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10963,6 +10963,9 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) this.ParseSubExpression(GetPrecedence(opKind))); } + // Check *explicitly* for `..` starting an expression. This *is* the initial term we want to parse out. + // If we have `expr..` though we don't do that here. Instead, we'll parse out 'expr', and the `..` + // portion will be handled in ParseExpressionContinued. if (tk == SyntaxKind.DotDotToken) { return _syntaxFactory.RangeExpression( @@ -11353,164 +11356,262 @@ private ExpressionSyntax ParseIsExpression(ExpressionSyntax leftOperand, SyntaxT } private ExpressionSyntax ParsePrimaryExpression(Precedence precedence) - => this.ParsePostFixExpression(ParseTermWithoutPostfix(precedence)); - - private ExpressionSyntax ParseTermWithoutPostfix(Precedence precedence) { - var tk = this.CurrentToken.Kind; - switch (tk) + // Primary expressions: + // x.y, f(x), a[i], x?.y, x?[y], x++, x--, x!, new, typeof, checked, unchecked, default, nameof, delegate, sizeof, stackalloc, x->y + // + // Note that postfix operators (like ++) are still primary expressions, even though their prefix equivalents (`++x`) are unary. + + return parsePostFixExpression(parsePrimaryExpressionWithoutPostfix(precedence)); + + ExpressionSyntax parsePrimaryExpressionWithoutPostfix(Precedence precedence) { - case SyntaxKind.TypeOfKeyword: - return this.ParseTypeOfExpression(); - case SyntaxKind.DefaultKeyword: - return this.ParseDefaultExpression(); - case SyntaxKind.SizeOfKeyword: - return this.ParseSizeOfExpression(); - case SyntaxKind.MakeRefKeyword: - return this.ParseMakeRefExpression(); - case SyntaxKind.RefTypeKeyword: - return this.ParseRefTypeExpression(); - case SyntaxKind.CheckedKeyword: - case SyntaxKind.UncheckedKeyword: - return this.ParseCheckedOrUncheckedExpression(); - case SyntaxKind.RefValueKeyword: - return this.ParseRefValueExpression(); - case SyntaxKind.ColonColonToken: - // misplaced :: - // Calling ParseAliasQualifiedName will cause us to create a missing identifier node that then - // properly consumes the :: and the reset of the alias name afterwards. - return this.ParseAliasQualifiedName(NameOptions.InExpression); - case SyntaxKind.EqualsGreaterThanToken: - return this.ParseLambdaExpression(); - case SyntaxKind.StaticKeyword: - if (this.IsPossibleAnonymousMethodExpression()) - { - return this.ParseAnonymousMethodExpression(); - } - else if (this.IsPossibleLambdaExpression(precedence)) - { + var tk = this.CurrentToken.Kind; + switch (tk) + { + case SyntaxKind.TypeOfKeyword: + return this.ParseTypeOfExpression(); + case SyntaxKind.DefaultKeyword: + return this.ParseDefaultExpression(); + case SyntaxKind.SizeOfKeyword: + return this.ParseSizeOfExpression(); + case SyntaxKind.MakeRefKeyword: + return this.ParseMakeRefExpression(); + case SyntaxKind.RefTypeKeyword: + return this.ParseRefTypeExpression(); + case SyntaxKind.CheckedKeyword: + case SyntaxKind.UncheckedKeyword: + return this.ParseCheckedOrUncheckedExpression(); + case SyntaxKind.RefValueKeyword: + return this.ParseRefValueExpression(); + case SyntaxKind.ColonColonToken: + // misplaced :: + // Calling ParseAliasQualifiedName will cause us to create a missing identifier node that then + // properly consumes the :: and the reset of the alias name afterwards. + return this.ParseAliasQualifiedName(NameOptions.InExpression); + case SyntaxKind.EqualsGreaterThanToken: return this.ParseLambdaExpression(); - } - else - { - return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, this.CurrentToken.Text); - } - case SyntaxKind.IdentifierToken: - { - if (this.IsTrueIdentifier()) + case SyntaxKind.StaticKeyword: + if (this.IsPossibleAnonymousMethodExpression()) { - if (this.IsPossibleAnonymousMethodExpression()) - { - return this.ParseAnonymousMethodExpression(); - } - else if (this.IsPossibleLambdaExpression(precedence) && this.TryParseLambdaExpression() is { } lambda) - { - return lambda; - } - else if (this.IsPossibleDeconstructionLeft(precedence)) - { - return ParseDeclarationExpression(ParseTypeMode.Normal, isScoped: false); - } - else if (IsCurrentTokenFieldInKeywordContext() && PeekToken(1).Kind != SyntaxKind.ColonColonToken) + return this.ParseAnonymousMethodExpression(); + } + else if (this.IsPossibleLambdaExpression(precedence)) + { + return this.ParseLambdaExpression(); + } + else + { + return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, this.CurrentToken.Text); + } + case SyntaxKind.IdentifierToken: + { + if (this.IsTrueIdentifier()) { - return _syntaxFactory.FieldExpression(this.EatContextualToken(SyntaxKind.FieldKeyword)); + if (this.IsPossibleAnonymousMethodExpression()) + { + return this.ParseAnonymousMethodExpression(); + } + else if (this.IsPossibleLambdaExpression(precedence) && this.TryParseLambdaExpression() is { } lambda) + { + return lambda; + } + else if (this.IsPossibleDeconstructionLeft(precedence)) + { + return ParseDeclarationExpression(ParseTypeMode.Normal, isScoped: false); + } + else if (IsCurrentTokenFieldInKeywordContext() && PeekToken(1).Kind != SyntaxKind.ColonColonToken) + { + return _syntaxFactory.FieldExpression(this.EatContextualToken(SyntaxKind.FieldKeyword)); + } + else + { + return this.ParseAliasQualifiedName(NameOptions.InExpression); + } } else { - return this.ParseAliasQualifiedName(NameOptions.InExpression); + return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, this.CurrentToken.Text); } } - else + case SyntaxKind.OpenBracketToken: + return this.IsPossibleLambdaExpression(precedence) + ? this.ParseLambdaExpression() + : this.ParseCollectionExpression(); + case SyntaxKind.ThisKeyword: + return _syntaxFactory.ThisExpression(this.EatToken()); + case SyntaxKind.BaseKeyword: + return ParseBaseExpression(); + + case SyntaxKind.ArgListKeyword: + case SyntaxKind.FalseKeyword: + case SyntaxKind.TrueKeyword: + case SyntaxKind.NullKeyword: + case SyntaxKind.NumericLiteralToken: + case SyntaxKind.StringLiteralToken: + case SyntaxKind.Utf8StringLiteralToken: + case SyntaxKind.SingleLineRawStringLiteralToken: + case SyntaxKind.Utf8SingleLineRawStringLiteralToken: + case SyntaxKind.MultiLineRawStringLiteralToken: + case SyntaxKind.Utf8MultiLineRawStringLiteralToken: + case SyntaxKind.CharacterLiteralToken: + return _syntaxFactory.LiteralExpression(SyntaxFacts.GetLiteralExpression(tk), this.EatToken()); + case SyntaxKind.InterpolatedStringStartToken: + case SyntaxKind.InterpolatedVerbatimStringStartToken: + case SyntaxKind.InterpolatedSingleLineRawStringStartToken: + case SyntaxKind.InterpolatedMultiLineRawStringStartToken: + throw new NotImplementedException(); // this should not occur because these tokens are produced and parsed immediately + case SyntaxKind.InterpolatedStringToken: + return this.ParseInterpolatedStringToken(); + case SyntaxKind.OpenParenToken: { - return this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, this.CurrentToken.Text); + return IsPossibleLambdaExpression(precedence) && this.TryParseLambdaExpression() is { } lambda + ? lambda + : this.ParseCastOrParenExpressionOrTuple(); } - } - case SyntaxKind.OpenBracketToken: - return this.IsPossibleLambdaExpression(precedence) - ? this.ParseLambdaExpression() - : this.ParseCollectionExpression(); - case SyntaxKind.ThisKeyword: - return _syntaxFactory.ThisExpression(this.EatToken()); - case SyntaxKind.BaseKeyword: - return ParseBaseExpression(); - - case SyntaxKind.ArgListKeyword: - case SyntaxKind.FalseKeyword: - case SyntaxKind.TrueKeyword: - case SyntaxKind.NullKeyword: - case SyntaxKind.NumericLiteralToken: - case SyntaxKind.StringLiteralToken: - case SyntaxKind.Utf8StringLiteralToken: - case SyntaxKind.SingleLineRawStringLiteralToken: - case SyntaxKind.Utf8SingleLineRawStringLiteralToken: - case SyntaxKind.MultiLineRawStringLiteralToken: - case SyntaxKind.Utf8MultiLineRawStringLiteralToken: - case SyntaxKind.CharacterLiteralToken: - return _syntaxFactory.LiteralExpression(SyntaxFacts.GetLiteralExpression(tk), this.EatToken()); - case SyntaxKind.InterpolatedStringStartToken: - case SyntaxKind.InterpolatedVerbatimStringStartToken: - case SyntaxKind.InterpolatedSingleLineRawStringStartToken: - case SyntaxKind.InterpolatedMultiLineRawStringStartToken: - throw new NotImplementedException(); // this should not occur because these tokens are produced and parsed immediately - case SyntaxKind.InterpolatedStringToken: - return this.ParseInterpolatedStringToken(); - case SyntaxKind.OpenParenToken: - { - return IsPossibleLambdaExpression(precedence) && this.TryParseLambdaExpression() is { } lambda - ? lambda - : this.ParseCastOrParenExpressionOrTuple(); - } - case SyntaxKind.NewKeyword: - return this.ParseNewExpression(); - case SyntaxKind.StackAllocKeyword: - return this.ParseStackAllocExpression(); - case SyntaxKind.DelegateKeyword: - // check for lambda expression with explicit function pointer return type - return this.IsPossibleLambdaExpression(precedence) - ? this.ParseLambdaExpression() - : this.ParseAnonymousMethodExpression(); - case SyntaxKind.RefKeyword: - // check for lambda expression with explicit ref return type: `ref int () => { ... }` - if (this.IsPossibleLambdaExpression(precedence)) - { - return this.ParseLambdaExpression(); - } - // ref is not expected to appear in this position. - var refKeyword = this.EatToken(); - return this.AddError(_syntaxFactory.RefExpression(refKeyword, this.ParseExpressionCore()), ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); - default: - if (IsPredefinedType(tk)) - { + case SyntaxKind.NewKeyword: + return this.ParseNewExpression(); + case SyntaxKind.StackAllocKeyword: + return this.ParseStackAllocExpression(); + case SyntaxKind.DelegateKeyword: + // check for lambda expression with explicit function pointer return type + return this.IsPossibleLambdaExpression(precedence) + ? this.ParseLambdaExpression() + : this.ParseAnonymousMethodExpression(); + case SyntaxKind.RefKeyword: + // check for lambda expression with explicit ref return type: `ref int () => { ... }` if (this.IsPossibleLambdaExpression(precedence)) { return this.ParseLambdaExpression(); } - - // check for intrinsic type followed by '.' - var expr = _syntaxFactory.PredefinedType(this.EatToken()); - - if (this.CurrentToken.Kind != SyntaxKind.DotToken || tk == SyntaxKind.VoidKeyword) + // ref is not expected to appear in this position. + var refKeyword = this.EatToken(); + return this.AddError(_syntaxFactory.RefExpression(refKeyword, this.ParseExpressionCore()), ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); + default: + if (IsPredefinedType(tk)) { - expr = this.AddError(expr, ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); - } + if (this.IsPossibleLambdaExpression(precedence)) + { + return this.ParseLambdaExpression(); + } - return expr; - } - else - { - var expr = this.CreateMissingIdentifierName(); + // check for intrinsic type followed by '.' + var expr = _syntaxFactory.PredefinedType(this.EatToken()); - if (tk == SyntaxKind.EndOfFileToken) - { - expr = this.AddError(expr, ErrorCode.ERR_ExpressionExpected); + if (this.CurrentToken.Kind != SyntaxKind.DotToken || tk == SyntaxKind.VoidKeyword) + { + expr = this.AddError(expr, ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); + } + + return expr; } else { - expr = this.AddError(expr, ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); + var expr = this.CreateMissingIdentifierName(); + + if (tk == SyntaxKind.EndOfFileToken) + { + expr = this.AddError(expr, ErrorCode.ERR_ExpressionExpected); + } + else + { + expr = this.AddError(expr, ErrorCode.ERR_InvalidExprTerm, SyntaxFacts.GetText(tk)); + } + + return expr; } + } + } - return expr; + ExpressionSyntax parsePostFixExpression(ExpressionSyntax expr) + { + Debug.Assert(expr != null); + + while (true) + { + // If the set of postfix expressions is updated here, please review ParseStatementAttributeDeclarations + // to see if it may need a similar look-ahead check to determine if something is a collection expression + // versus an attribute. + + switch (this.CurrentToken.Kind) + { + case SyntaxKind.OpenParenToken: + expr = _syntaxFactory.InvocationExpression(expr, this.ParseParenthesizedArgumentList()); + continue; + + case SyntaxKind.OpenBracketToken: + expr = _syntaxFactory.ElementAccessExpression(expr, this.ParseBracketedArgumentList()); + continue; + + case SyntaxKind.PlusPlusToken: + case SyntaxKind.MinusMinusToken: + expr = _syntaxFactory.PostfixUnaryExpression(SyntaxFacts.GetPostfixUnaryExpression(this.CurrentToken.Kind), expr, this.EatToken()); + continue; + + case SyntaxKind.ColonColonToken: + if (this.PeekToken(1).Kind == SyntaxKind.IdentifierToken) + { + expr = _syntaxFactory.MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + expr, + // replace :: with missing dot and annotate with skipped text "::" and error + this.ConvertToMissingWithTrailingTrivia(this.AddError(this.EatToken(), ErrorCode.ERR_UnexpectedAliasedName), SyntaxKind.DotToken), + this.ParseSimpleName(NameOptions.InExpression)); + } + else + { + // just some random trailing :: ? + expr = AddTrailingSkippedSyntax(expr, this.EatTokenWithPrejudice(SyntaxKind.DotToken)); + } + + continue; + + case SyntaxKind.MinusGreaterThanToken: + expr = _syntaxFactory.MemberAccessExpression(SyntaxKind.PointerMemberAccessExpression, expr, this.EatToken(), this.ParseSimpleName(NameOptions.InExpression)); + continue; + + case SyntaxKind.DotToken: + // if we have the error situation: + // + // expr. + // X Y + // + // Then we don't want to parse this out as "Expr.X" + // + // It's far more likely the member access expression is simply incomplete and + // there is a new declaration on the next line. + if (this.CurrentToken.TrailingTrivia.Any((int)SyntaxKind.EndOfLineTrivia) && + this.PeekToken(1).Kind == SyntaxKind.IdentifierToken && + this.PeekToken(2).ContextualKind == SyntaxKind.IdentifierToken) + { + return _syntaxFactory.MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, expr, this.EatToken(), + this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_IdentifierExpected)); + } + + expr = _syntaxFactory.MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, expr, this.EatToken(), this.ParseSimpleName(NameOptions.InExpression)); + continue; + + case SyntaxKind.QuestionToken: + if (CanStartConsequenceExpression()) + { + expr = _syntaxFactory.ConditionalAccessExpression( + expr, + this.EatToken(), + ParseConsequenceSyntax()); + continue; + } + + return expr; + + case SyntaxKind.ExclamationToken: + expr = _syntaxFactory.PostfixUnaryExpression(SyntaxKind.SuppressNullableWarningExpression, expr, this.EatToken()); + continue; + + default: + return expr; } + } } } @@ -11594,97 +11695,6 @@ private bool IsPossibleAnonymousMethodExpression() this.PeekToken(tokenIndex + 1).Kind != SyntaxKind.AsteriskToken; } - private ExpressionSyntax ParsePostFixExpression(ExpressionSyntax expr) - { - Debug.Assert(expr != null); - - while (true) - { - // If the set of postfix expressions is updated here, please review ParseStatementAttributeDeclarations - // to see if it may need a similar look-ahead check to determine if something is a collection expression - // versus an attribute. - - switch (this.CurrentToken.Kind) - { - case SyntaxKind.OpenParenToken: - expr = _syntaxFactory.InvocationExpression(expr, this.ParseParenthesizedArgumentList()); - continue; - - case SyntaxKind.OpenBracketToken: - expr = _syntaxFactory.ElementAccessExpression(expr, this.ParseBracketedArgumentList()); - continue; - - case SyntaxKind.PlusPlusToken: - case SyntaxKind.MinusMinusToken: - expr = _syntaxFactory.PostfixUnaryExpression(SyntaxFacts.GetPostfixUnaryExpression(this.CurrentToken.Kind), expr, this.EatToken()); - continue; - - case SyntaxKind.ColonColonToken: - if (this.PeekToken(1).Kind == SyntaxKind.IdentifierToken) - { - expr = _syntaxFactory.MemberAccessExpression( - SyntaxKind.SimpleMemberAccessExpression, - expr, - // replace :: with missing dot and annotate with skipped text "::" and error - this.ConvertToMissingWithTrailingTrivia(this.AddError(this.EatToken(), ErrorCode.ERR_UnexpectedAliasedName), SyntaxKind.DotToken), - this.ParseSimpleName(NameOptions.InExpression)); - } - else - { - // just some random trailing :: ? - expr = AddTrailingSkippedSyntax(expr, this.EatTokenWithPrejudice(SyntaxKind.DotToken)); - } - - continue; - - case SyntaxKind.MinusGreaterThanToken: - expr = _syntaxFactory.MemberAccessExpression(SyntaxKind.PointerMemberAccessExpression, expr, this.EatToken(), this.ParseSimpleName(NameOptions.InExpression)); - continue; - - case SyntaxKind.DotToken: - // if we have the error situation: - // - // expr. - // X Y - // - // Then we don't want to parse this out as "Expr.X" - // - // It's far more likely the member access expression is simply incomplete and - // there is a new declaration on the next line. - if (this.CurrentToken.TrailingTrivia.Any((int)SyntaxKind.EndOfLineTrivia) && - this.PeekToken(1).Kind == SyntaxKind.IdentifierToken && - this.PeekToken(2).ContextualKind == SyntaxKind.IdentifierToken) - { - return _syntaxFactory.MemberAccessExpression( - SyntaxKind.SimpleMemberAccessExpression, expr, this.EatToken(), - this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_IdentifierExpected)); - } - - expr = _syntaxFactory.MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, expr, this.EatToken(), this.ParseSimpleName(NameOptions.InExpression)); - continue; - - case SyntaxKind.QuestionToken: - if (CanStartConsequenceExpression()) - { - expr = _syntaxFactory.ConditionalAccessExpression( - expr, - this.EatToken(), - ParseConsequenceSyntax()); - continue; - } - - return expr; - - case SyntaxKind.ExclamationToken: - expr = _syntaxFactory.PostfixUnaryExpression(SyntaxKind.SuppressNullableWarningExpression, expr, this.EatToken()); - continue; - - default: - return expr; - } - } - } - private bool CanStartConsequenceExpression() { Debug.Assert(this.CurrentToken.Kind == SyntaxKind.QuestionToken); From 16497d4acb9931bac97c1ecddaee7f8b50a3bb47 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 18 Oct 2024 11:00:13 -0700 Subject: [PATCH 29/44] Simplify token merging --- .../CSharp/Portable/Parser/LanguageParser.cs | 195 ++++++++++-------- 1 file changed, 108 insertions(+), 87 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 40246f5d3a3e3..251e817244af1 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11048,66 +11048,13 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence // Look for operators that can follow what we've seen so far, and which are acceptable at this // precedence level. Examples include binary operator, assignment operators, range operators `..`, as // well as `switch` and `with` clauses. - var tk = this.CurrentToken.ContextualKind; - SyntaxKind opKind; + var (operatorTokenKind, operatorExpressionKind) = getOperatorTokenAndExpressionKind(); - // If the set of expression continuations is updated here, please review ParseStatementAttributeDeclarations - // to see if it may need a similar look-ahead check to determine if something is a collection expression versus - // an attribute. - - // check for >>, >>=, >>> or >>>= - if (tk == SyntaxKind.GreaterThanToken - && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token1 - && NoTriviaBetween(this.CurrentToken, token1)) // check to see if they really are adjacent - { - if (token1.Kind == SyntaxKind.GreaterThanToken) - { - if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 - && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent - { - opKind = token2.Kind == SyntaxKind.GreaterThanToken - ? SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanToken) - : SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); - } - else - { - opKind = SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken); - } - } - else - { - opKind = SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken); - } - } - else if (IsExpectedBinaryOperator(tk)) - { - opKind = SyntaxFacts.GetBinaryExpression(tk); - } - else if (IsExpectedAssignmentOperator(tk)) - { - opKind = SyntaxFacts.GetAssignmentExpression(tk); - } - else if (tk == SyntaxKind.DotDotToken) - { - opKind = SyntaxKind.RangeExpression; - } - else if (tk == SyntaxKind.SwitchKeyword && this.PeekToken(1).Kind == SyntaxKind.OpenBraceToken) - { - opKind = SyntaxKind.SwitchExpression; - } - else if (tk == SyntaxKind.WithKeyword && this.PeekToken(1).Kind == SyntaxKind.OpenBraceToken) - { - opKind = SyntaxKind.WithExpression; - } - else - { - // Something that doesn't expand the current expression we're looking at. Bail out and see if we - // can end with a conditional expression. + if (operatorTokenKind == SyntaxKind.None) return false; - } - var newPrecedence = GetPrecedence(opKind); + var newPrecedence = GetPrecedence(operatorExpressionKind); // Check the precedence to see if we should "take" this operator. A lower precedence means what's // coming isn't a child of us, but rather we will be a child of it. So we bail out and let any higher @@ -11116,12 +11063,12 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence return false; // Same precedence, but not right-associative -- deal with this "later" - if ((newPrecedence == precedence) && !IsRightAssociative(opKind)) + if ((newPrecedence == precedence) && !IsRightAssociative(operatorExpressionKind)) return false; - // We'll "take" this operator, as precedence is tentatively OK. + // No consume the operator (including consuming multiple tokens in the case of merged operator tokens) - var operatorToken = eatOperatorToken(opKind); + var operatorToken = eatOperatorToken(operatorTokenKind); if (newPrecedence > GetPrecedence(leftOperand.Kind)) { @@ -11142,24 +11089,24 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence operatorToken.Text); } - if (opKind == SyntaxKind.AsExpression) + if (operatorExpressionKind == SyntaxKind.AsExpression) { leftOperand = _syntaxFactory.BinaryExpression( - opKind, leftOperand, operatorToken, this.ParseType(ParseTypeMode.AsExpression)); + operatorExpressionKind, leftOperand, operatorToken, this.ParseType(ParseTypeMode.AsExpression)); } - else if (opKind == SyntaxKind.IsExpression) + else if (operatorExpressionKind == SyntaxKind.IsExpression) { leftOperand = ParseIsExpression(leftOperand, operatorToken); } - else if (opKind == SyntaxKind.SwitchExpression) + else if (operatorExpressionKind == SyntaxKind.SwitchExpression) { leftOperand = ParseSwitchExpression(leftOperand, operatorToken); } - else if (opKind == SyntaxKind.WithExpression) + else if (operatorExpressionKind == SyntaxKind.WithExpression) { leftOperand = ParseWithExpression(leftOperand, operatorToken); } - else if (opKind == SyntaxKind.RangeExpression) + else if (operatorExpressionKind == SyntaxKind.RangeExpression) { leftOperand = _syntaxFactory.RangeExpression( leftOperand, @@ -11168,11 +11115,11 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence ? this.ParseSubExpression(Precedence.Range) : null); } - else if (SyntaxFacts.IsAssignmentExpression(opKind)) + else if (IsExpectedAssignmentOperator(operatorToken.Kind)) { ExpressionSyntax rhs; - if (opKind == SyntaxKind.SimpleAssignmentExpression && CurrentToken.Kind == SyntaxKind.RefKeyword && + if (operatorExpressionKind == SyntaxKind.SimpleAssignmentExpression && CurrentToken.Kind == SyntaxKind.RefKeyword && // check for lambda expression with explicit ref return type: `ref int () => { ... }` !this.IsPossibleLambdaExpression(newPrecedence)) { @@ -11185,11 +11132,13 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence rhs = this.ParseSubExpression(newPrecedence); } - leftOperand = _syntaxFactory.AssignmentExpression(opKind, leftOperand, operatorToken, rhs); + leftOperand = _syntaxFactory.AssignmentExpression( + operatorExpressionKind, leftOperand, operatorToken, rhs); } - else if (IsExpectedBinaryOperator(tk)) + else if (IsExpectedBinaryOperator(operatorToken.Kind)) { - leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); + leftOperand = _syntaxFactory.BinaryExpression( + operatorExpressionKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); } else { @@ -11199,35 +11148,107 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence return true; } - SyntaxToken eatOperatorToken(SyntaxKind operatorKind) + (SyntaxKind operatorTokenKind, SyntaxKind operatorExpressionKind) getOperatorTokenAndExpressionKind() + { + // If the set of expression continuations is updated here, please review ParseStatementAttributeDeclarations + // to see if it may need a similar look-ahead check to determine if something is a collection expression versus + // an attribute. + + var token1 = this.CurrentToken; + var token1Kind = token1.Kind; + + // check for >>, >>=, >>> or >>>= + if (token1Kind == SyntaxKind.GreaterThanToken + && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 + && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent + { + if (token2.Kind == SyntaxKind.GreaterThanToken) + { + // >> or >>> or >>= + + if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token3 + && NoTriviaBetween(token2, token3)) // check to see if they really are adjacent + { + // >>> or >>>= + + var tokenKind = token3.Kind == SyntaxKind.GreaterThanToken + ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken + : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; + + var operatorKind = token3.Kind == SyntaxKind.GreaterThanToken + ? SyntaxFacts.GetBinaryExpression(tokenKind) + : SyntaxFacts.GetAssignmentExpression(tokenKind); + return (tokenKind, operatorKind); + } + else + { + // >> + return (SyntaxKind.GreaterThanGreaterThanToken, SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken)); + } + } + else + { + // >>= + return (SyntaxKind.GreaterThanGreaterThanEqualsToken, SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken)); + } + + // Fall through. Just a normal `>` which will be handled by IsExpectedBinaryOperator below + } + + if (IsExpectedBinaryOperator(token1Kind)) + return (token1Kind, SyntaxFacts.GetBinaryExpression(token1Kind)); + + if (IsExpectedAssignmentOperator(token1Kind)) + return (token1Kind, SyntaxFacts.GetAssignmentExpression(token1Kind)); + + if (token1Kind == SyntaxKind.DotDotToken) + return (token1Kind, SyntaxKind.RangeExpression); + + if (token1Kind == SyntaxKind.SwitchKeyword && this.PeekToken(1).Kind == SyntaxKind.OpenBraceToken) + return (token1Kind, SyntaxKind.SwitchExpression); + + if (token1Kind == SyntaxKind.WithKeyword && this.PeekToken(1).Kind == SyntaxKind.OpenBraceToken) + return (token1Kind, SyntaxKind.WithExpression); + + // Something that doesn't expand the current expression we're looking at. Bail out and see if we + // can end with a conditional expression. + return (SyntaxKind.None, SyntaxKind.None); + } + + SyntaxToken eatOperatorToken(SyntaxKind operatorTokenKind) { // Combine tokens into a single token if needed - if (operatorKind is SyntaxKind.RightShiftExpression or SyntaxKind.RightShiftAssignmentExpression) + if (operatorTokenKind is SyntaxKind.GreaterThanGreaterThanToken or SyntaxKind.GreaterThanGreaterThanEqualsToken) { - // >> or >>= - return SyntaxFactory.Token( - this.EatToken().GetLeadingTrivia(), - operatorKind == SyntaxKind.RightShiftExpression ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken, - this.EatToken().GetTrailingTrivia()); + // >> and >>= + // Two tokens need to be consumed here. + var token1 = EatToken(); + var token2 = EatToken(); + + return SyntaxFactory.Token( + token1.GetLeadingTrivia(), + operatorTokenKind, + token2.GetTrailingTrivia()); } - else if (operatorKind is SyntaxKind.UnsignedRightShiftExpression or SyntaxKind.UnsignedRightShiftAssignmentExpression) + else if (operatorTokenKind is SyntaxKind.GreaterThanGreaterThanGreaterThanToken or SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken) { - // >>> or >>>= - var leadingTrivia = this.EatToken().GetLeadingTrivia(); + // >>> and >>>= + // Three tokens need to be consumed here. - // Skip middle token. - _ = this.EatToken(); + var token1 = EatToken(); + _ = EatToken(); + var token3 = EatToken(); return SyntaxFactory.Token( - leadingTrivia, - operatorKind == SyntaxKind.UnsignedRightShiftExpression ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, - this.EatToken().GetTrailingTrivia()); + token1.GetLeadingTrivia(), + operatorTokenKind, + token3.GetTrailingTrivia()); } else { - // Normal operator - return this.EatContextualToken(this.CurrentToken.ContextualKind); + // Normal operator. Eat as a single token (converting cases contextual words 'with' to a keyword) + return this.EatContextualToken(operatorTokenKind); } } From c5f6fb704fe88c4f9c8925587f8caed5ffe3c52b Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 18 Oct 2024 11:02:26 -0700 Subject: [PATCH 30/44] Fix --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 251e817244af1..ed771a5e797fc 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11155,7 +11155,7 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence // an attribute. var token1 = this.CurrentToken; - var token1Kind = token1.Kind; + var token1Kind = token1.ContextualKind; // check for >>, >>=, >>> or >>>= if (token1Kind == SyntaxKind.GreaterThanToken From c19fd49272c3db5ffefdc7362313ce6dcd46e679 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 18 Oct 2024 11:04:14 -0700 Subject: [PATCH 31/44] Simplify --- .../CSharp/Portable/Parser/LanguageParser.cs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index ed771a5e797fc..e71a2e7bca773 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11158,41 +11158,33 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence var token1Kind = token1.ContextualKind; // check for >>, >>=, >>> or >>>= + // + // In all those cases, update token1Kind to be the merged token kind. It will then be handled by the code below. if (token1Kind == SyntaxKind.GreaterThanToken && this.PeekToken(1) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token2 && NoTriviaBetween(token1, token2)) // check to see if they really are adjacent { if (token2.Kind == SyntaxKind.GreaterThanToken) { - // >> or >>> or >>= - if (this.PeekToken(2) is { Kind: SyntaxKind.GreaterThanToken or SyntaxKind.GreaterThanEqualsToken } token3 && NoTriviaBetween(token2, token3)) // check to see if they really are adjacent { // >>> or >>>= - - var tokenKind = token3.Kind == SyntaxKind.GreaterThanToken + token1Kind = token3.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; - - var operatorKind = token3.Kind == SyntaxKind.GreaterThanToken - ? SyntaxFacts.GetBinaryExpression(tokenKind) - : SyntaxFacts.GetAssignmentExpression(tokenKind); - return (tokenKind, operatorKind); } else { // >> - return (SyntaxKind.GreaterThanGreaterThanToken, SyntaxFacts.GetBinaryExpression(SyntaxKind.GreaterThanGreaterThanToken)); + token1Kind = SyntaxKind.GreaterThanGreaterThanToken; } } else { // >>= - return (SyntaxKind.GreaterThanGreaterThanEqualsToken, SyntaxFacts.GetAssignmentExpression(SyntaxKind.GreaterThanGreaterThanEqualsToken)); + token1Kind = SyntaxKind.GreaterThanGreaterThanEqualsToken; } - - // Fall through. Just a normal `>` which will be handled by IsExpectedBinaryOperator below } if (IsExpectedBinaryOperator(token1Kind)) From 6dcfc98cd1ce0185959af0264964596e2f329c79 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 18 Oct 2024 11:12:58 -0700 Subject: [PATCH 32/44] Simplify --- .../CSharp/Portable/Parser/LanguageParser.cs | 69 +++++++++---------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index e71a2e7bca773..0f5f7fd3c864d 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11013,6 +11013,8 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) } } +#nullable enable + /// /// Takes in an initial unary expression or primary expression, and then consumes what follows as long as as /// it's precedence is either lower than the we're parsing currently (or equal, to @@ -11020,11 +11022,11 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) /// private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimaryExpression, Precedence precedence) { - var expandedExpression = unaryOrPrimaryExpression; + var currentExpression = unaryOrPrimaryExpression; // Keep on expanding the left operand as long as what we see fits the precedence we're under. - while (tryExpandExpression(ref expandedExpression, precedence)) - continue; + while (tryExpandExpression(currentExpression, precedence) is ExpressionSyntax expandedExpression) + currentExpression = expandedExpression; // Finally, consume a conditional expression if precedence allows it. @@ -11039,11 +11041,11 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimar // 2. When parsing the branches of the expression, parse at the highest precedence again ('expression'). // This allows for things like assignments/lambdas in the branches of the conditional. if (this.CurrentToken.Kind == SyntaxKind.QuestionToken && precedence <= Precedence.Conditional) - return consumeConditionalExpression(expandedExpression); + return consumeConditionalExpression(currentExpression); - return expandedExpression; + return currentExpression; - bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence) + ExpressionSyntax? tryExpandExpression(ExpressionSyntax leftOperand, Precedence precedence) { // Look for operators that can follow what we've seen so far, and which are acceptable at this // precedence level. Examples include binary operator, assignment operators, range operators `..`, as @@ -11052,7 +11054,7 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence var (operatorTokenKind, operatorExpressionKind) = getOperatorTokenAndExpressionKind(); if (operatorTokenKind == SyntaxKind.None) - return false; + return null; var newPrecedence = GetPrecedence(operatorExpressionKind); @@ -11060,11 +11062,11 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence // coming isn't a child of us, but rather we will be a child of it. So we bail out and let any higher // up expression parsing consume it with us as the left side. if (newPrecedence < precedence) - return false; + return null; // Same precedence, but not right-associative -- deal with this "later" if ((newPrecedence == precedence) && !IsRightAssociative(operatorExpressionKind)) - return false; + return null; // No consume the operator (including consuming multiple tokens in the case of merged operator tokens) @@ -11091,31 +11093,30 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence if (operatorExpressionKind == SyntaxKind.AsExpression) { - leftOperand = _syntaxFactory.BinaryExpression( + return _syntaxFactory.BinaryExpression( operatorExpressionKind, leftOperand, operatorToken, this.ParseType(ParseTypeMode.AsExpression)); } - else if (operatorExpressionKind == SyntaxKind.IsExpression) - { - leftOperand = ParseIsExpression(leftOperand, operatorToken); - } - else if (operatorExpressionKind == SyntaxKind.SwitchExpression) - { - leftOperand = ParseSwitchExpression(leftOperand, operatorToken); - } - else if (operatorExpressionKind == SyntaxKind.WithExpression) - { - leftOperand = ParseWithExpression(leftOperand, operatorToken); - } - else if (operatorExpressionKind == SyntaxKind.RangeExpression) + + if (operatorExpressionKind == SyntaxKind.IsExpression) + return ParseIsExpression(leftOperand, operatorToken); + + if (operatorExpressionKind == SyntaxKind.SwitchExpression) + return ParseSwitchExpression(leftOperand, operatorToken); + + if (operatorExpressionKind == SyntaxKind.WithExpression) + return ParseWithExpression(leftOperand, operatorToken); + + if (operatorExpressionKind == SyntaxKind.RangeExpression) { - leftOperand = _syntaxFactory.RangeExpression( + return _syntaxFactory.RangeExpression( leftOperand, operatorToken, CanStartExpression() ? this.ParseSubExpression(Precedence.Range) : null); } - else if (IsExpectedAssignmentOperator(operatorToken.Kind)) + + if (IsExpectedAssignmentOperator(operatorToken.Kind)) { ExpressionSyntax rhs; @@ -11132,20 +11133,14 @@ bool tryExpandExpression(ref ExpressionSyntax leftOperand, Precedence precedence rhs = this.ParseSubExpression(newPrecedence); } - leftOperand = _syntaxFactory.AssignmentExpression( + return _syntaxFactory.AssignmentExpression( operatorExpressionKind, leftOperand, operatorToken, rhs); } - else if (IsExpectedBinaryOperator(operatorToken.Kind)) - { - leftOperand = _syntaxFactory.BinaryExpression( - operatorExpressionKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); - } - else - { - throw ExceptionUtilities.Unreachable(); - } - return true; + if (IsExpectedBinaryOperator(operatorToken.Kind)) + return _syntaxFactory.BinaryExpression(operatorExpressionKind, leftOperand, operatorToken, this.ParseSubExpression(newPrecedence)); + + throw ExceptionUtilities.Unreachable(); } (SyntaxKind operatorTokenKind, SyntaxKind operatorExpressionKind) getOperatorTokenAndExpressionKind() @@ -11338,6 +11333,8 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) } } +#nullable restore + private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) { var scopedKeyword = isScoped From 1976c0039b9eb20a40869af2290b5564e0a2a35c Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 18 Oct 2024 12:53:59 -0700 Subject: [PATCH 33/44] Simplify --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 0f5f7fd3c864d..dcc65f3c0e3c1 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11333,7 +11333,7 @@ static bool containsTernaryCollectionToReinterpret(ExpressionSyntax expression) } } -#nullable restore +#nullable disable private DeclarationExpressionSyntax ParseDeclarationExpression(ParseTypeMode mode, bool isScoped) { From 7ff892060a5a7467420276190f798abe1f08b4bd Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 22 Oct 2024 09:10:05 -0700 Subject: [PATCH 34/44] Update src/Compilers/CSharp/Portable/Parser/LanguageParser.cs Co-authored-by: Rikki Gibson --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index dcc65f3c0e3c1..16036ce55224e 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11068,7 +11068,7 @@ private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimar if ((newPrecedence == precedence) && !IsRightAssociative(operatorExpressionKind)) return null; - // No consume the operator (including consuming multiple tokens in the case of merged operator tokens) + // Now consume the operator (including consuming multiple tokens in the case of merged operator tokens) var operatorToken = eatOperatorToken(operatorTokenKind); From 804f7a66cf9d1950d93b29a014249661ce1859ae Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 11:50:27 -0700 Subject: [PATCH 35/44] Tweak comment --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 16036ce55224e..27c158c4f7826 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11017,7 +11017,7 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) /// /// Takes in an initial unary expression or primary expression, and then consumes what follows as long as as - /// it's precedence is either lower than the we're parsing currently (or equal, to + /// it's precedence is either lower than the we're parsing currently, or equal to /// that precedence if we have something right-associative . /// private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimaryExpression, Precedence precedence) From caf82c5996f9c502cacc29ccfaa95e2c6c8d6ac6 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 11:50:57 -0700 Subject: [PATCH 36/44] Grammar --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 27c158c4f7826..e12d22d90671c 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11016,9 +11016,9 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) #nullable enable /// - /// Takes in an initial unary expression or primary expression, and then consumes what follows as long as as - /// it's precedence is either lower than the we're parsing currently, or equal to - /// that precedence if we have something right-associative . + /// Takes in an initial unary expression or primary expression, and then consumes what follows as long as its + /// precedence is either lower than the we're parsing currently, or equal to that + /// precedence if we have something right-associative . /// private ExpressionSyntax ParseExpressionContinued(ExpressionSyntax unaryOrPrimaryExpression, Precedence precedence) { From 52190a9698431353a6cd42826cb13d27e7535d7e Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 12:42:19 -0700 Subject: [PATCH 37/44] Update src/Compilers/CSharp/Portable/Parser/LanguageParser.cs --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index e12d22d90671c..80aa9f19c9a19 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -11234,7 +11234,7 @@ SyntaxToken eatOperatorToken(SyntaxKind operatorTokenKind) } else { - // Normal operator. Eat as a single token (converting cases contextual words 'with' to a keyword) + // Normal operator. Eat as a single token, converting contextual words cases (like 'with') to a keyword. return this.EatContextualToken(operatorTokenKind); } } From cc4ce546f969f31a6ca1a24846eb5966aba2f888 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 12:45:34 -0700 Subject: [PATCH 38/44] Lower value slightly --- src/Compilers/CSharp/Test/EndToEnd/EndToEndTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Test/EndToEnd/EndToEndTests.cs b/src/Compilers/CSharp/Test/EndToEnd/EndToEndTests.cs index 8add99ff84be8..3ecdfa9c4a313 100644 --- a/src/Compilers/CSharp/Test/EndToEnd/EndToEndTests.cs +++ b/src/Compilers/CSharp/Test/EndToEnd/EndToEndTests.cs @@ -413,7 +413,7 @@ public void NestedIfStatements() int nestingLevel = (IntPtr.Size, ExecutionConditionUtil.Configuration) switch { (4, ExecutionConfiguration.Debug) => 310, - (4, ExecutionConfiguration.Release) => 1419, + (4, ExecutionConfiguration.Release) => 1400, (8, ExecutionConfiguration.Debug) => 200, (8, ExecutionConfiguration.Release) => 474, _ => throw new Exception($"Unexpected configuration {IntPtr.Size * 8}-bit {ExecutionConditionUtil.Configuration}") From 7bb8097e9fa86b3254a54100bbe80d0f4e78382f Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 12:59:04 -0700 Subject: [PATCH 39/44] Fix --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index 38bf6ae4974ea..e9fc0ee7fea7b 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10938,7 +10938,7 @@ ExpressionSyntax parseUnaryOrPrimaryExpression(Precedence precedence) { return _syntaxFactory.RangeExpression( leftOperand: null, - this.EatToken(), + this.EatDotDotToken(), CanStartExpression() ? this.ParseSubExpression(Precedence.Range) : null); From cf6235b401341618c4aeac3ea01060b0065f585e Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 13:10:37 -0700 Subject: [PATCH 40/44] Clean up usings --- .../Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs index 0c6d1eb9a28a0..4d4ed05966982 100644 --- a/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/IncrementalParsing/IncrementalParsingTests.cs @@ -7,12 +7,9 @@ using System; using System.Collections.Immutable; using System.Linq; -using ICSharpCode.Decompiler.TypeSystem; -using System.Reflection; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.CSharp.Test.Utilities; using Microsoft.CodeAnalysis.Text; -using Newtonsoft.Json.Linq; using Roslyn.Test.Utilities; using Xunit; using Xunit.Abstractions; From 6dc3f54df4b9c2db471c56f41aeae6ea63f3f12a Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 23 Oct 2024 13:21:12 -0700 Subject: [PATCH 41/44] Add tests --- .../Parsing/RangeExpressionParsingTests.cs | 224 ++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs new file mode 100644 index 0000000000000..829529ce5dc9a --- /dev/null +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs @@ -0,0 +1,224 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.CodeAnalysis.CSharp.UnitTests; + +public sealed class RangeExpressionParsingTests(ITestOutputHelper output) + : ParsingTests(output) +{ + [Fact] + public void CastingRangeExpressionWithoutStartOrEnd() + { + UsingExpression("(int).."); + + N(SyntaxKind.CastExpression); + { + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.PredefinedType); + { + N(SyntaxKind.IntKeyword); + } + N(SyntaxKind.CloseParenToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + } + } + EOF(); + } + + [Fact] + public void CastingRangeExpressionWithoutStart() + { + UsingExpression("(int)..0"); + + N(SyntaxKind.CastExpression); + { + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.PredefinedType); + { + N(SyntaxKind.IntKeyword); + } + N(SyntaxKind.CloseParenToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + N(SyntaxKind.NumericLiteralExpression); + { + N(SyntaxKind.NumericLiteralToken, "0"); + } + } + } + EOF(); + } + + [Fact] + public void ConditionalExpressionWithEmptyRangeForWhenTrue() + { + UsingExpression("a ? .. : b"); + + N(SyntaxKind.ConditionalExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "a"); + } + N(SyntaxKind.QuestionToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + } + N(SyntaxKind.ColonToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "b"); + } + } + EOF(); + } + + [Fact] + public void ConditionalExpressionWithEmptyRangeForWhenFalse() + { + UsingExpression("a ? b : .."); + + N(SyntaxKind.ConditionalExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "a"); + } + N(SyntaxKind.QuestionToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "b"); + } + N(SyntaxKind.ColonToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + } + } + EOF(); + } + + [Fact] + public void ConditionalExpressionWithEmptyRangeForWhenTrueAndWhenFalse() + { + UsingExpression("a ? .. : .."); + + N(SyntaxKind.ConditionalExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "a"); + } + N(SyntaxKind.QuestionToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + } + N(SyntaxKind.ColonToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + } + } + EOF(); + } + + [Fact] + public void ConditionalExpressionWithEmptyStartRangeForWhenTrue() + { + UsingExpression("a ? ..b : c"); + + N(SyntaxKind.ConditionalExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "a"); + } + N(SyntaxKind.QuestionToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "b"); + } + } + N(SyntaxKind.ColonToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "c"); + } + } + EOF(); + } + + [Fact] + public void ConditionalExpressionWithEmptyStartRangeForWhenFalse() + { + UsingExpression("a ? b : ..c"); + + N(SyntaxKind.ConditionalExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "a"); + } + N(SyntaxKind.QuestionToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "b"); + } + N(SyntaxKind.ColonToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "c"); + } + } + } + EOF(); + } + + [Fact] + public void ConditionalExpressionWithEmptyStartRangeForWhenTrueAndFalse() + { + UsingExpression("a ? ..b : ..c"); + + N(SyntaxKind.ConditionalExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "a"); + } + N(SyntaxKind.QuestionToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "b"); + } + } + N(SyntaxKind.ColonToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "c"); + } + } + } + EOF(); + } +} From a9b971092a4f06f3a936976d43fd9a3f0b870ff0 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 24 Oct 2024 19:59:04 -0700 Subject: [PATCH 42/44] Simplify --- src/Compilers/CSharp/Portable/Parser/LanguageParser.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index e9fc0ee7fea7b..d96514ff2769c 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -7845,6 +7845,8 @@ private SyntaxList ParseStatementAttributeDeclarations() // Check the next token to see if it indicates the `[...]` sequence we have is a term or not. This is the // same set of tokens that ParsePostFixExpression looks for. + // + // Note `SyntaxKind.DotToken` handles both the `[...].Name` case as well as the `[...]..Range` case. var isCollectionExpression = this.CurrentToken.Kind is SyntaxKind.DotToken or SyntaxKind.QuestionToken @@ -7860,7 +7862,6 @@ or SyntaxKind.MinusMinusToken isCollectionExpression = isCollectionExpression || IsExpectedBinaryOperator(this.CurrentToken.Kind) || IsExpectedAssignmentOperator(this.CurrentToken.Kind) - || IsAtDotDotToken() || (this.CurrentToken.ContextualKind is SyntaxKind.SwitchKeyword or SyntaxKind.WithKeyword && this.PeekToken(1).Kind is SyntaxKind.OpenBraceToken); if (!isCollectionExpression && From d86ef0d44b8a2e20f90efca4cc2cea06451a0e69 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 24 Oct 2024 20:15:54 -0700 Subject: [PATCH 43/44] Add tests --- .../Parsing/RangeExpressionParsingTests.cs | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs index 829529ce5dc9a..9d7c48edf940d 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs @@ -221,4 +221,67 @@ public void ConditionalExpressionWithEmptyStartRangeForWhenTrueAndFalse() } EOF(); } + + [Fact] + public void CastingRangeExpressionInPattern1() + { + UsingExpression("x is (int).."); + + N(SyntaxKind.IsPatternExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "x"); + } + N(SyntaxKind.IsKeyword); + N(SyntaxKind.ConstantPattern); + { + N(SyntaxKind.CastExpression); + { + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.PredefinedType); + { + N(SyntaxKind.IntKeyword); + } + N(SyntaxKind.CloseParenToken); + N(SyntaxKind.RangeExpression); + { + N(SyntaxKind.DotDotToken); + } + } + } + } + EOF(); + } + + [Fact] + public void CastingRangeExpressionInPattern2() + { + UsingExpression("x is (int).", + // (1,1): error CS1073: Unexpected token '.' + // x is (int). + Diagnostic(ErrorCode.ERR_UnexpectedToken, "x is (int)").WithArguments(".").WithLocation(1, 1)); + + N(SyntaxKind.IsPatternExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "x"); + } + N(SyntaxKind.IsKeyword); + N(SyntaxKind.ParenthesizedPattern); + { + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.TypePattern); + { + N(SyntaxKind.PredefinedType); + { + N(SyntaxKind.IntKeyword); + } + } + N(SyntaxKind.CloseParenToken); + } + } + EOF(); + } } From ff0e6e4d8a8717b7b7b9fd331a6ceb9005f1ab6a Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 24 Oct 2024 20:28:23 -0700 Subject: [PATCH 44/44] Add tests --- .../Parsing/RangeExpressionParsingTests.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs index 9d7c48edf940d..3a3fc86d16063 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/RangeExpressionParsingTests.cs @@ -284,4 +284,35 @@ public void CastingRangeExpressionInPattern2() } EOF(); } + + [Fact] + public void CastingRangeExpressionInPattern3() + { + UsingExpression("x is (int).Length", + // (1,1): error CS1073: Unexpected token '.' + // x is (int).Length + Diagnostic(ErrorCode.ERR_UnexpectedToken, "x is (int)").WithArguments(".").WithLocation(1, 1)); + + N(SyntaxKind.IsPatternExpression); + { + N(SyntaxKind.IdentifierName); + { + N(SyntaxKind.IdentifierToken, "x"); + } + N(SyntaxKind.IsKeyword); + N(SyntaxKind.ParenthesizedPattern); + { + N(SyntaxKind.OpenParenToken); + N(SyntaxKind.TypePattern); + { + N(SyntaxKind.PredefinedType); + { + N(SyntaxKind.IntKeyword); + } + } + N(SyntaxKind.CloseParenToken); + } + } + EOF(); + } }