diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb b/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb index e01a47a99bf26..dc241af8f0957 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary Option Strict On @@ -139,7 +139,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return node.GetTrailingTrivia().Any(SyntaxKind.EndOfLineTrivia) Case SyntaxKind.SingleLineIfStatement, SyntaxKind.SingleLineElseClause - ' Steer clear of single-line if's because they they have custom handling of statement + ' Steer clear of single-line if's because they they have custom handling of statement ' terminators that may make it difficult to reuse sub-statements. Return False Case Else @@ -301,7 +301,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Shared Function ShouldCrumble(node As VisualBasicSyntaxNode) As Boolean If TypeOf node Is StructuredTriviaSyntax Then ' Do not crumble into structured trivia content. - ' we will not use any of the parts anyways and + ' we will not use any of the parts anyways and ' evaluation of directives may go out of sync. Return False End If @@ -516,14 +516,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax _currentToken = _currentToken.With(nextPreprocessorState) End If - ' this will discard any prefetched tokens, including current. + ' this will discard any prefetched tokens, including current. ' We do not need them since we moved to completely new node. MyBase.MoveToNextSyntaxNode() TryPopNode() ' at this point, all three pointers (position in old tree, position in text, position in parser) - ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState) + ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState) ' should point to same position (in sync) End Sub @@ -537,7 +537,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' just move forward _lineBufferOffset = _lineBufferOffset + _curNodeLength - ' this will just verify that we do not have any prefetched tokens, including current. + ' this will just verify that we do not have any prefetched tokens, including current. ' otherwise advancing linebuffer offeset could go out of sync with token stream. MyBase.MoveToNextSyntaxNodeInTrivia() diff --git a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb index 099a979f87b7c..159ab2da46bf4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Strict On Option Explicit On @@ -339,7 +339,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic End Function ''' - ''' Creates a half width form Unicode character string. + ''' Creates a half width form Unicode character string. ''' ''' The text representing the original identifier. This can be in full width or half width Unicode form. ''' A string representing the text in a half width Unicode form. @@ -488,4 +488,38 @@ Namespace Microsoft.CodeAnalysis.VisualBasic End Function End Class + Friend Module CharExts + + + Friend Function IsFrom(c As Char, c0 As Char, c1 As Char) As Boolean + Return (c = c0) OrElse (c = c1) + End Function + + + Friend Function IsFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Return (c = c0) OrElse (c = c1) OrElse (c = c2) + End Function + + + Friend Function IsFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Return (c = c0) OrElse (c = c1) OrElse (c = c2) OrElse (c = c3) + End Function + + + Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char) As Boolean + Return (c <> c0) AndAlso (c <> c1) + End Function + + + Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) + End Function + + + Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) AndAlso (c <> c3) + End Function + + End Module + End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb b/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb index 29ef1692c75c2..f8aaa56b2c5eb 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb @@ -31,8 +31,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim restorePoint = CreateRestorePoint() Me._isScanningDirective = True - ' since we do not have lookahead tokens, this just - ' resets current token to _lineBufferOffset + ' since we do not have lookahead tokens, this just + ' resets current token to _lineBufferOffset Me.GetNextTokenInState(ScannerState.VB) Dim currentNonterminal = Me.GetCurrentSyntaxNode() @@ -43,7 +43,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Me.MoveToNextSyntaxNodeInTrivia() ' adjust current token to just after the node - ' we need that in case we need to skip some disabled text + ' we need that in case we need to skip some disabled text '(yes we do tokenize disabled text for compatibility reasons) Me.GetNextTokenInState(ScannerState.VB) @@ -91,10 +91,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax disabledCode = SkipConditionalCompilationSection() End If - ' Here we add the directive and disabled text that follows it + ' Here we add the directive and disabled text that follows it ' (if there is any) to the trivia list - ' processing statement could add an error to it, + ' processing statement could add an error to it, ' so we may need to rebuild the trivia node If statement IsNot directiveTrivia Then directiveTrivia = statement @@ -507,7 +507,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '// '// Skip all text until the end of the current conditional section. This will also return the '// span of text that was skipped - '// + '// '//------------------------------------------------------------------------------------------------- Private Function SkipConditionalCompilationSection() As SyntaxList(Of VisualBasicSyntaxNode) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb index 47997330eb86d..bd05eb43d7068 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb @@ -269,7 +269,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax Public kdIsQueryClause As Boolean '// Note: "CanFollowExpr" says whether this token can come after an expression. - '// e.g. "Dim x = From i In Select i" is valid, therefore "Select" can come follow an expression. + '// e.g. "Dim x = From i In Select i" is valid, therefore "Select" can come follow an expression. '// The complete list was discovered through the tool in vb\Language\Tools\VBGrammarAnalyzer\vbgrammar.html '// If you add keywords, then make sure that they're added to the official language grammar, and re-run the tool. Public kdCanFollowExpr As Boolean @@ -304,9 +304,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax Private Shared Function EnsureHalfWidth(s As String) As String Dim result As Char() = Nothing - For i As Integer = 0 To s.Length - 1 - Dim ch = s(i) + Dim ch As Char = s(i) If SyntaxFacts.IsFullWidth(ch) Then ch = SyntaxFacts.MakeHalfWidth(ch) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb b/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb index 298c37f2058d5..32a0f8819a232 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb @@ -16,16 +16,16 @@ Imports Microsoft.CodeAnalysis.VisualBasic.Syntax Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '' The QuickTokenAccumulator is a small mini-tokenizer that may fail. It consumes characters and '' eventually either decides that either it found a complete token (including the trivia on either - '' side), or it gives up and says that the full scanner should take an attempt. It also accumulates the + '' side), or it gives up and says that the full scanner should take an attempt. It also accumulates the '' token into a character buffer and computes a hash code. The entire tokenization is done by a single '' routine without any memory allocations to keep it very fast. - '' + '' '' Currently it only handles two cases: '' optional-whitespace keyword-or-identifier optional-whitespace '' optional-whitespace single-char-punctuation optional-whitespace - '' + '' '' where the whitespace does not include newlines. - ' + '' '' The VB tokenization rules are complex, and care needs to be taken in constructing the quick '' tokenization. For example "REM" begins a comment, so it can't be tokenized as a keyword or identifier. '' Similar problems arise with multi-character punctuation tokens, which can have embedded spaces. @@ -63,7 +63,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax 'TODO: why : and ; are complex? (8th row, 3 and 4) - ' The following table classifies the first &H180 Unicode characters. + ' The following table classifies the first &H180 Unicode characters. ' R and r are marked as COMPLEX so that quick-scanning doesn't stop after "REM". ' # is marked complex as it may start directives. ' PERF: Use UShort instead of CharFlags so the compiler can use array literal initialization. @@ -163,7 +163,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' Attempt to scan a single token. ' If it succeeds, return True, and the characters, length, and hashcode of the token - ' can be retrieved by other functions. + ' can be retrieved by other functions. ' If it fails (the token is too complex), return False. Public Function QuickScanToken(allowLeadingMultilineTrivia As Boolean) As QuickScanResult Dim state As AccumulatorState = If(allowLeadingMultilineTrivia, AccumulatorState.InitialAllowLeadingMultilineTrivia, AccumulatorState.Initial) @@ -330,7 +330,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax index += 1 - 'FNV-like hash should work here + 'FNV-like hash should work here 'since these strings are short and mostly ASCII hashCode = (hashCode Xor unicodeValue) * Hash.FnvPrime End While diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index b669b950320a3..10885abcaec81 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary @@ -37,8 +37,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private ReadOnly _sbPooled As PooledStringBuilder = PooledStringBuilder.GetInstance ''' - ''' DO NOT USE DIRECTLY. - ''' USE GetScratch() + ''' DO NOT USE DIRECTLY. + ''' USE GetScratch() ''' Private ReadOnly _sb As StringBuilder = _sbPooled.Builder Private ReadOnly _triviaListPool As New SyntaxListPool @@ -90,7 +90,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetScratch() As StringBuilder ' the normal pattern is that we clean scratch after use. - ' hitting this assert very likely indicates that you + ' hitting this assert very likely indicates that you ' did not release scratch content or worse trying to use ' scratch in two places at a time. Debug.Assert(_sb.Length = 0, "trying to use dirty buffer?") @@ -157,7 +157,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function GetNextToken(Optional allowLeadingMultilineTrivia As Boolean = False) As SyntaxToken - ' Use quick token scanning to see if we can scan a token quickly. + ' Use quick token scanning to see if we can scan a token quickly. Dim quickToken = QuickScanToken(allowLeadingMultilineTrivia) If quickToken.Succeeded Then @@ -225,16 +225,16 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanNextCharAsToken(leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim token As SyntaxToken - - If Not CanGet() Then + Dim c As Char + If Not Peep(c) Then token = MakeEofToken(leadingTrivia) Else _badTokenCount += 1 If _badTokenCount < BadTokenCountLimit Then ' // Don't break up surrogate pairs - Dim c = Peek() - Dim length = If(IsHighSurrogate(c) AndAlso CanGet(1) AndAlso IsLowSurrogate(Peek(1)), 2, 1) + c = Peek() + Dim length = If(IsHighSurrogate(c) AndAlso Peep(1, c) AndAlso IsLowSurrogate(c), 2, 1) token = MakeBadToken(leadingTrivia, length, ERRID.ERR_IllegalChar) Else ' If we get too many characters that we cannot make sense of, absorb the rest of the input. @@ -265,9 +265,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Dim condLineStart = _lineBufferOffset - - While (CanGet()) - Dim c As Char = Peek() + Dim c As Char + While Peep(c) Select Case (c) @@ -281,12 +280,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax EatWhitespace() Continue While - Case _ - "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, - "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, - "y"c, "z"c, "A"c, "B"c, "C"c, "D"c, "E"c, "F"c, "G"c, "H"c, "I"c, "J"c, - "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, "R"c, "S"c, "T"c, "U"c, "V"c, - "W"c, "X"c, "Y"c, "Z"c, "'"c, "_"c + Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, + "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, + "y"c, "z"c, "A"c, "B"c, "C"c, "D"c, "E"c, "F"c, "G"c, "H"c, "I"c, "J"c, + "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, "R"c, "S"c, "T"c, "U"c, "V"c, + "W"c, "X"c, "Y"c, "Z"c, "'"c, "_"c EatThroughLine() condLineStart = _lineBufferOffset @@ -322,23 +320,22 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Sub EatThroughLine() - While CanGet() - Dim c As Char = Peek() - - If IsNewLine(c) Then - EatThroughLineBreak(c) - Return - Else - AdvanceChar() - End If - End While + Dim c As Char + While Peep(c) + If IsNewLine(c) Then + EatThroughLineBreak(c) + Return + Else + AdvanceChar() + End If + End While End Sub ''' ''' Gets a chunk of text as a DisabledCode node. ''' ''' The range of text. - ''' The DisabledCode node. + ''' The DisabledCode node. Friend Function GetDisabledTextAt(span As TextSpan) As SyntaxTrivia If span.Start >= 0 AndAlso span.End <= _bufferLen Then Return SyntaxFactory.DisabledTextTrivia(GetTextNotInterned(span.Start, span.Length)) @@ -487,11 +484,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function LengthOfLineBreak(StartCharacter As Char, Optional here As Integer = 0) As Integer - Debug.Assert(CanGet(here)) +#If DEBUG Then + Dim c As Char + Dim res = Peep(here, c) + Debug.Assert(res) Debug.Assert(IsNewLine(StartCharacter)) - - Debug.Assert(StartCharacter = Peek(here)) - + Debug.Assert(StartCharacter = c) +#End If If StartCharacter = CARRIAGE_RETURN AndAlso NextIs(here + 1, LINE_FEED) Then Return 2 End If @@ -503,7 +502,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Accept a CR/LF pair or either in isolation as a newline. ''' Make it a statement separator - ''' + ''' Private Function ScanNewlineAsStatementTerminator(startCharacter As Char, precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken If _lineBufferOffset < _endOfTerminatorTrivia Then Dim width = LengthOfLineBreak(startCharacter) @@ -533,27 +532,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanLineContinuation(tList As SyntaxListBuilder) As Boolean - If Not CanGet() Then - Return False - End If - - If Not IsAfterWhitespace() Then - Return False - End If - - Dim ch As Char = Peek() - If Not IsUnderscore(ch) Then + Dim ch As Char + If Not Peep(ch) OrElse Not IsAfterWhitespace() OrElse Not IsUnderscore(ch) Then Return False End If Dim Here = 1 - While CanGet(Here) - ch = Peek(Here) - If IsWhitespace(ch) Then - Here += 1 - Else - Exit While - End If + While Peep(Here, ch) AndAlso IsWhitespace(ch) + Here += 1 End While ' Line continuation is valid at the end of the @@ -578,10 +564,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' do not include the new line character since that would confuse code handling ' implicit line continuations. (See Scanner::EatLineContinuation.) Otherwise, ' include the new line and any additional spaces as trivia. - If startComment = 0 AndAlso - CanGet(Here) AndAlso - Not IsNewLine(Peek(Here)) Then + If startComment = 0 AndAlso Peep(Here, ch) AndAlso Not IsNewLine(ch) Then tList.Add(MakeEndOfLineTrivia(GetText(newLine))) If spaces > 0 Then tList.Add(MakeWhiteSpaceTrivia(GetText(spaces))) @@ -601,15 +585,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Consumes all trivia until a nontrivia char is found ''' Friend Function ScanMultilineTrivia() As SyntaxList(Of VisualBasicSyntaxNode) - If Not CanGet() Then + Dim ch As Char + If Not Peep(ch) Then Return Nothing End If - Dim ch = Peek() - ' optimization for a common case ' the ASCII range between ': and ~ , with exception of except "'", "_" and R cannot start trivia - If ch > ":"c AndAlso ch <= "~"c AndAlso ch <> "'"c AndAlso ch <> "_"c AndAlso ch <> "R"c AndAlso ch <> "r"c Then + If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsnotFrom("'"c, "_"c, "R"c, "r"c) Then Return Nothing End If @@ -626,7 +609,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Scans a single piece of trivia ''' Private Function TryScanSinglePieceOfMultilineTrivia(tList As SyntaxListBuilder) As Boolean - If CanGet() Then + Dim ch As Char + If Peep(ch) Then Dim atNewLine = IsAtNewLine() @@ -641,7 +625,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If End If - Dim ch = Peek() If IsWhitespace(ch) Then ' eat until linebreak or nonwhitespace Dim wslen = GetWhitespaceLength(1) @@ -686,11 +669,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' check for # Private Function StartsDirective(Here As Integer) As Boolean - If CanGet(Here) Then - Dim ch = Peek(Here) - Return IsHash(ch) - End If - Return False + Dim ch As Char + Return Peep(Here, ch) AndAlso IsHash(ch) End Function Private Function IsAtNewLine() As Boolean @@ -730,8 +710,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Sub Private Sub ScanSingleLineTriviaInXmlDoc(tList As SyntaxListBuilder) - If CanGet() Then - Dim c As Char = Peek() + Dim c As Char + If Peep(c) Then Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -763,7 +743,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Sub ScanWhitespaceAndLineContinuations(tList As SyntaxListBuilder) - If CanGet() AndAlso IsWhitespace(Peek()) Then + Dim c As Char + If Peep(c) AndAlso IsWhitespace(c) Then tList.Add(ScanWhitespace(1)) ' collect { lineCont, ws } While ScanLineContinuation(tList) @@ -830,12 +811,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' 3. colon followed by new line -> colon terminator + new line terminator ' 4. new line followed by new line -> new line terminator + new line terminator - ' Case 3 is required to parse single line if's and numeric labels. + ' Case 3 is required to parse single line if's and numeric labels. ' Case 4 is required to limit explicit line continuations to single new line + Dim ch As Char + If Peep(ch) Then - If CanGet() Then - - Dim ch As Char = Peek() Dim startOfTerminatorTrivia = _lineBufferOffset If IsNewLine(ch) Then @@ -847,11 +827,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' collect { ws, colon } Do Dim len = GetWhitespaceLength(0) - If Not CanGet(len) Then + If Not Peep(len, ch) Then Exit Do End If - ch = Peek(len) If Not IsColonAndNotColonEquals(ch, offset:=len) Then Exit Do End If @@ -887,7 +866,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetWhitespaceLength(len As Integer) As Integer ' eat until linebreak or nonwhitespace - While CanGet(len) AndAlso IsWhitespace(Peek(len)) + Dim c As Char + While Peep(len, c) AndAlso IsWhitespace(c) len += 1 End While Return len @@ -895,7 +875,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetXmlWhitespaceLength(len As Integer) As Integer ' eat until linebreak or nonwhitespace - While CanGet(len) AndAlso IsXmlWhitespace(Peek(len)) + Dim c As Char + While Peep(len, c) AndAlso IsXmlWhitespace(c) len += 1 End While Return len @@ -918,32 +899,32 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Sub EatWhitespace() - Debug.Assert(CanGet) - Debug.Assert(IsWhitespace(Peek())) - + Dim c As Char +#If DEBUG Then + Dim ok = Peep(c) + Debug.Assert(ok) + Debug.Assert(IsWhitespace(c)) +#End If AdvanceChar() - ' eat until linebreak or nonwhitespace - While CanGet() AndAlso IsWhitespace(Peek) + While Peep(c) AndAlso IsWhitespace(c) AdvanceChar() End While End Sub Private Function PeekStartComment(i As Integer) As Integer - - If CanGet(i) Then - Dim ch = Peek(i) - + Dim ch As Char + If Peep(i, ch) Then If IsSingleQuote(ch) Then Return 1 - ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso - CanGet(i + 2) AndAlso MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso - MatchOneOrAnotherOrFullwidth(Peek(i + 2), "M"c, "m"c) Then + ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso Peep(i + 2, ch) AndAlso + MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso + MatchOneOrAnotherOrFullwidth(ch, "M"c, "m"c) Then - If Not CanGet(i + 3) OrElse IsNewLine(Peek(i + 3)) Then + If Not Peep(i + 3, ch) OrElse IsNewLine(ch) Then ' have only 'REM' Return 3 - ElseIf Not IsIdentifierPartCharacter(Peek(i + 3)) Then + ElseIf Not IsIdentifierPartCharacter(ch) Then ' have 'REM ' Return 4 End If @@ -961,9 +942,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim looksLikeDocComment As Boolean = StartsXmlDoc(0) ' eat all chars until EoL - While CanGet(length) AndAlso - Not IsNewLine(Peek(length)) - + Dim c As Char + While Peep(length, c) AndAlso Not IsNewLine(c) length += 1 End While @@ -995,450 +975,191 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #End Region - ' at this point it is very likely that we are located at - ' the beginning of a token - Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - - If Not CanGet() Then - Return MakeEofToken(precedingTrivia) - End If + ' at this point it is very likely that we are located at + ' the beginning of a token + Private Function ScanTokenCommon(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char, fullWidth As Boolean) As SyntaxToken + Dim lengthWithMaybeEquals As Integer = 1 + Dim c As Char - Dim ch As Char = Peek() Select Case ch - Case CARRIAGE_RETURN, LINE_FEED, NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR + Case CARRIAGE_RETURN, LINE_FEED Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) - + Case NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR + If Not fullWidth Then + Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) + End If Case " "c, CHARACTER_TABULATION, "'"c Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch))) Return Nothing ' trivia cannot start a token - Case "@"c - Return MakeAtToken(precedingTrivia, False) - + Return MakeAtToken(precedingTrivia, fullWidth) Case "("c - Return MakeOpenParenToken(precedingTrivia, False) - + Return MakeOpenParenToken(precedingTrivia, fullWidth) Case ")"c - Return MakeCloseParenToken(precedingTrivia, False) - + Return MakeCloseParenToken(precedingTrivia, fullWidth) Case "{"c - Return MakeOpenBraceToken(precedingTrivia, False) - + Return MakeOpenBraceToken(precedingTrivia, fullWidth) Case "}"c - Return MakeCloseBraceToken(precedingTrivia, False) - + Return MakeCloseBraceToken(precedingTrivia, fullWidth) Case ","c - Return MakeCommaToken(precedingTrivia, False) - + Return MakeCommaToken(precedingTrivia, fullWidth) Case "#"c Dim dl = ScanDateLiteral(precedingTrivia) - If dl IsNot Nothing Then - Return dl - Else - Return MakeHashToken(precedingTrivia, False) - End If - + Return If(dl, MakeHashToken(precedingTrivia, fullWidth)) Case "&"c - If CanGet(1) AndAlso BeginsBaseLiteral(Peek(1)) Then + If Peep(1, c) AndAlso BeginsBaseLiteral(c) Then Return ScanNumericLiteral(precedingTrivia) End If - - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeAmpersandEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeAmpersandToken(precedingTrivia, False) + Return MakeAmpersandToken(precedingTrivia, fullWidth) End If - Case "="c - Return MakeEqualsToken(precedingTrivia, False) - + Return MakeEqualsToken(precedingTrivia, fullWidth) Case "<"c - Return ScanLeftAngleBracket(precedingTrivia, False, _scanSingleLineTriviaFunc) - + Return ScanLeftAngleBracket(precedingTrivia, fullWidth, _scanSingleLineTriviaFunc) Case ">"c - Return ScanRightAngleBracket(precedingTrivia, False) - + Return ScanRightAngleBracket(precedingTrivia, fullWidth) Case ":"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeColonEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return ScanColonAsStatementTerminator(precedingTrivia, False) + Return ScanColonAsStatementTerminator(precedingTrivia, fullWidth) End If - Case "+"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakePlusEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakePlusToken(precedingTrivia, False) + Return MakePlusToken(precedingTrivia, fullWidth) End If - Case "-"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeMinusEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeMinusToken(precedingTrivia, False) + Return MakeMinusToken(precedingTrivia, fullWidth) End If - Case "*"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeAsteriskEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeAsteriskToken(precedingTrivia, False) + Return MakeAsteriskToken(precedingTrivia, fullWidth) End If - Case "/"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeSlashToken(precedingTrivia, False) + Return MakeSlashToken(precedingTrivia, fullWidth) End If - Case "\"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeBackSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeBackslashToken(precedingTrivia, False) + Return MakeBackslashToken(precedingTrivia, fullWidth) End If - Case "^"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeCaretEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeCaretToken(precedingTrivia, False) + Return MakeCaretToken(precedingTrivia, fullWidth) End If - Case "!"c - Return MakeExclamationToken(precedingTrivia, False) - + Return MakeExclamationToken(precedingTrivia, fullWidth) Case "."c If CanGet(1) AndAlso IsDecimalDigit(Peek(1)) Then Return ScanNumericLiteral(precedingTrivia) Else - Return MakeDotToken(precedingTrivia, False) + Return MakeDotToken(precedingTrivia, fullWidth) End If - - Case "0"c, - "1"c, - "2"c, - "3"c, - "4"c, - "5"c, - "6"c, - "7"c, - "8"c, - "9"c + Case "0"c, "1"c, "2"c, "3"c, "4"c, + "5"c, "6"c, "7"c, "8"c, "9"c Return ScanNumericLiteral(precedingTrivia) - Case """"c Return ScanStringLiteral(precedingTrivia) - Case "A"c If NextAre(1, "s ") Then - ' TODO: do we allow widechars in keywords? - Dim spelling = "As" AdvanceChar(2) - Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia) + Return MakeKeyword(SyntaxKind.AsKeyword, "As", precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If - Case "E"c If NextAre(1, "nd ") Then - ' TODO: do we allow widechars in keywords? - Dim spelling = "End" AdvanceChar(3) - Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia) + Return MakeKeyword(SyntaxKind.EndKeyword, "End", precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If - Case "I"c If NextAre(1, "f ") Then - ' TODO: do we allow widechars in keywords? - Dim spelling = "If" AdvanceChar(2) - Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia) + Return MakeKeyword(SyntaxKind.IfKeyword, "If", precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If - Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, "y"c, "z"c Return ScanIdentifierOrKeyword(precedingTrivia) - Case "B"c, "C"c, "D"c, "F"c, "G"c, "H"c, "J"c, "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, - "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c + "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c Return ScanIdentifierOrKeyword(precedingTrivia) - Case "_"c - If CanGet(1) AndAlso IsIdentifierPartCharacter(Peek(1)) Then + If Peep(1, c) AndAlso IsIdentifierPartCharacter(c) Then Return ScanIdentifierOrKeyword(precedingTrivia) End If - Dim err As ERRID = ERRID.ERR_ExpectedIdentifier Dim len = GetWhitespaceLength(1) If Not CanGet(len) OrElse IsNewLine(Peek(len)) OrElse PeekStartComment(len) > 0 Then err = ERRID.ERR_LineContWithCommentOrNoPrecSpace End If - ' not a line continuation and cannot start identifier. Return MakeBadToken(precedingTrivia, 1, err) - Case "["c Return ScanBracketedIdentifier(precedingTrivia) - Case "?"c - Return MakeQuestionToken(precedingTrivia, False) - + Return MakeQuestionToken(precedingTrivia, fullWidth) Case "%"c - If CanGet(1) AndAlso - Peek(1) = ">"c Then + If NextIs(1, ">"c) Then Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc) End If - Case "$"c, FULLWIDTH_DOLLAR_SIGN - If CanGet(1) AndAlso IsDoubleQuote(Peek(1)) Then + If Not fullWidth AndAlso Peep(1, c) AndAlso IsDoubleQuote(c) Then Return MakePunctuationToken(precedingTrivia, 2, SyntaxKind.DollarSignDoubleQuoteToken) End If - End Select - If IsIdentifierStartCharacter(ch) Then Return ScanIdentifierOrKeyword(precedingTrivia) End If - Debug.Assert(Not IsNewLine(ch)) - + If fullWidth Then + Debug.Assert(Not IsDoubleQuote(ch)) + Return nothing + End If If IsDoubleQuote(ch) Then - Return ScanStringLiteral(precedingTrivia) + Return ScanStringLiteral(precedingTrivia) End If - If IsFullWidth(ch) Then ch = MakeHalfWidth(ch) Return ScanTokenFullWidth(precedingTrivia, ch) End If - Return Nothing End Function - ' REVIEW: Is there a better way to reuse this logic? - Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken - Select Case ch - Case CARRIAGE_RETURN, LINE_FEED - Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) - - Case " "c, CHARACTER_TABULATION, "'"c - Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch))) - Return Nothing ' trivia cannot start a token - - Case "@"c - Return MakeAtToken(precedingTrivia, True) - - Case "("c - Return MakeOpenParenToken(precedingTrivia, True) - - Case ")"c - Return MakeCloseParenToken(precedingTrivia, True) - - Case "{"c - Return MakeOpenBraceToken(precedingTrivia, True) - - Case "}"c - Return MakeCloseBraceToken(precedingTrivia, True) - - Case ","c - Return MakeCommaToken(precedingTrivia, True) - - Case "#"c - Dim dl = ScanDateLiteral(precedingTrivia) - If dl IsNot Nothing Then - Return dl - Else - Return MakeHashToken(precedingTrivia, True) - End If - - Case "&"c - If CanGet(1) AndAlso BeginsBaseLiteral(Peek(1)) Then - Return ScanNumericLiteral(precedingTrivia) - End If - - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeAmpersandEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeAmpersandToken(precedingTrivia, True) - End If - - Case "="c - Return MakeEqualsToken(precedingTrivia, True) - - Case "<"c - Return ScanLeftAngleBracket(precedingTrivia, True, _scanSingleLineTriviaFunc) - - Case ">"c - Return ScanRightAngleBracket(precedingTrivia, True) - - Case ":"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeColonEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return ScanColonAsStatementTerminator(precedingTrivia, True) - End If - - Case "+"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakePlusEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakePlusToken(precedingTrivia, True) - End If - - Case "-"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeMinusEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeMinusToken(precedingTrivia, True) - End If - - Case "*"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeAsteriskEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeAsteriskToken(precedingTrivia, True) - End If - - Case "/"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeSlashToken(precedingTrivia, True) - End If - - Case "\"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeBackSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeBackslashToken(precedingTrivia, True) - End If - - Case "^"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeCaretEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeCaretToken(precedingTrivia, True) - End If - - Case "!"c - Return MakeExclamationToken(precedingTrivia, True) - - Case "."c - If CanGet(1) AndAlso IsDecimalDigit(Peek(1)) Then - Return ScanNumericLiteral(precedingTrivia) - Else - Return MakeDotToken(precedingTrivia, True) - End If - - Case "0"c, - "1"c, - "2"c, - "3"c, - "4"c, - "5"c, - "6"c, - "7"c, - "8"c, - "9"c - Return ScanNumericLiteral(precedingTrivia) - - Case """"c - Return ScanStringLiteral(precedingTrivia) - - Case "A"c - If NextAre(1, "s ") Then - Dim spelling = GetText(2) - Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia) - Else - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Case "E"c - If NextAre(1, "nd ") Then - Dim spelling = GetText(3) - Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia) - Else - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Case "I"c - If NextAre(1, "f ") Then - ' TODO: do we allow widechars in keywords? - Dim spelling = GetText(2) - Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia) - Else - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, "m"c, - "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, "y"c, "z"c - Return ScanIdentifierOrKeyword(precedingTrivia) - - Case "B"c, "C"c, "D"c, "F"c, "G"c, "H"c, "J"c, "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, - "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c - Return ScanIdentifierOrKeyword(precedingTrivia) - - Case "_"c - If CanGet(1) AndAlso IsIdentifierPartCharacter(Peek(1)) Then - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Dim err As ERRID = ERRID.ERR_ExpectedIdentifier - Dim len = GetWhitespaceLength(1) - If Not CanGet(len) OrElse IsNewLine(Peek(len)) OrElse PeekStartComment(len) > 0 Then - err = ERRID.ERR_LineContWithCommentOrNoPrecSpace - End If - - ' not a line continuation and cannot start identifier. - Return MakeBadToken(precedingTrivia, 1, err) - - Case "["c - Return ScanBracketedIdentifier(precedingTrivia) - - Case "?"c - Return MakeQuestionToken(precedingTrivia, True) - - Case "%"c - If CanGet(1) AndAlso - Peek(1) = ">"c Then - Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc) - End If - - End Select - - If IsIdentifierStartCharacter(ch) Then - Return ScanIdentifierOrKeyword(precedingTrivia) + ' at this point it is very likely that we are located at + ' the beginning of a token + Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken + Dim ch As Char + If Not Peep(ch) Then + Return MakeEofToken(precedingTrivia) End If + Return ScanTokenCommon(precedingTrivia, ch, False) + End Function - Debug.Assert(Not IsNewLine(ch)) - Debug.Assert(Not IsDoubleQuote(ch)) - - Return Nothing + Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken + Return ScanTokenCommon(precedingTrivia, ch, True) End Function ' // Allow whitespace between the characters of a two-character token. @@ -1449,11 +1170,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim Here = Index Dim eq As Char - While CanGet(Here) - eq = Peek(Here) + While Peep(Here, eq) Here += 1 If Not IsWhitespace(eq) Then - If eq = "="c OrElse eq = FULLWIDTH_EQUALS_SIGN Then + If eq.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then Index = Here Return True Else @@ -1465,21 +1185,23 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanRightAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean) As SyntaxToken - Debug.Assert(CanGet) ' > - Debug.Assert(Peek() = ">"c OrElse Peek() = FULLWIDTH_GREATER_THAN_SIGN) + Dim c As Char +#If DEBUG Then + Dim ok = Peep(c) + Debug.Assert(ok) ' > + Debug.Assert(c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN)) +#End If Dim length As Integer = 1 ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - If CanGet(length) Then - Dim c As Char = Peek(length) - - If c = "="c OrElse c = FULLWIDTH_EQUALS_SIGN Then + If Peep(length, c) Then + If c.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 Return MakeGreaterThanEqualsToken(precedingTrivia, length) - ElseIf c = ">"c OrElse c = FULLWIDTH_GREATER_THAN_SIGN Then + ElseIf c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then length += 1 If TrySkipFollowingEquals(length) Then Return MakeGreaterThanGreaterThanEqualsToken(precedingTrivia, length) @@ -1492,33 +1214,31 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanLeftAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean, scanTrailingTrivia As ScanTriviaFunc) As SyntaxToken - Debug.Assert(CanGet) ' < - Debug.Assert(Peek() = "<"c OrElse Peek() = FULLWIDTH_LESS_THAN_SIGN) - + Dim c As Char +#If DEBUG Then + Dim ok = Peep(c) + Debug.Assert(ok) ' < + Debug.Assert(c.IsFrom("<"c, FULLWIDTH_LESS_THAN_SIGN)) +#End If Dim length As Integer = 1 - ' Check for XML tokens - If Not charIsFullWidth AndAlso CanGet(length) Then - Dim c As Char = Peek(length) + If Not charIsFullWidth AndAlso Peep(length, c) Then Select Case c Case "!"c - If CanGet(length + 2) Then + If Peep(length + 2, c) Then Select Case (Peek(length + 1)) Case "-"c - If CanGet(length + 3) AndAlso Peek(length + 2) = "-"c Then + If CanGet(length + 3) AndAlso c = "-"c Then Return XmlMakeBeginCommentToken(precedingTrivia, scanTrailingTrivia) End If Case "["c - If NextAre(length + 2, "CDATA[") Then - Return XmlMakeBeginCDataToken(precedingTrivia, scanTrailingTrivia) End If End Select End If Case "?"c Return XmlMakeBeginProcessingInstructionToken(precedingTrivia, scanTrailingTrivia) - Case "/"c Return XmlMakeBeginEndElementToken(precedingTrivia, _scanSingleLineTriviaFunc) End Select @@ -1526,24 +1246,18 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - - If CanGet(length) Then - Dim c As Char = Peek(length) - - If c = "="c OrElse c = FULLWIDTH_EQUALS_SIGN Then + If Peep(length, c) Then + If c.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 Return MakeLessThanEqualsToken(precedingTrivia, length) - ElseIf c = ">"c OrElse c = FULLWIDTH_GREATER_THAN_SIGN Then + ElseIf c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then length += 1 Return MakeLessThanGreaterThanToken(precedingTrivia, length) - ElseIf c = "<"c OrElse c = FULLWIDTH_LESS_THAN_SIGN Then + ElseIf c.IsFrom("<"c, FULLWIDTH_LESS_THAN_SIGN) Then length += 1 - - If CanGet(length) Then - c = Peek(length) - + If Peep(length, c) Then 'if the second "<" is a part of "<%" - like in "<<%" , we do not want to use it. - If c <> "%"c AndAlso c <> FULLWIDTH_PERCENT_SIGN Then + If c.IsnotFrom("%"c, FULLWIDTH_PERCENT_SIGN) Then If TrySkipFollowingEquals(length) Then Return MakeLessThanLessThanEqualsToken(precedingTrivia, length) Else @@ -1568,10 +1282,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim c = spelling(0) If SyntaxFacts.IsIdentifierStartCharacter(c) Then - ' SPEC: ... Visual Basic identifiers conform to the Unicode Standard Annex 15 with one - ' SPEC: exception: identifiers may begin with an underscore (connector) character. - ' SPEC: If an identifier begins with an underscore, it must contain at least one other - ' SPEC: valid identifier character to disambiguate it from a line continuation. + ' SPEC: ... Visual Basic identifiers conform to the Unicode Standard Annex 15 with one + ' SPEC: exception: identifiers may begin with an underscore (connector) character. + ' SPEC: If an identifier begins with an underscore, it must contain at least one other + ' SPEC: valid identifier character to disambiguate it from a line continuation. If IsConnectorPunctuation(c) AndAlso spellingLength = 1 Then Return False End If @@ -1587,14 +1301,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanIdentifierOrKeyword(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Debug.Assert(CanGet) - Debug.Assert(IsIdentifierStartCharacter(Peek)) - Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller + Dim ch, cx As Char + Dim ok = Peep(ch) ' Non DEBUG usage see:= If IsConnectorPunctuation(ch) - Dim ch = Peek() - If CanGet(1) Then - Dim ch1 = Peek(1) - If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(ch1) Then + Debug.Assert(ok) + Debug.Assert(IsIdentifierStartCharacter(ch)) + Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller + If Peep(1, cx) Then + If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(cx) Then Return MakeBadToken(precedingTrivia, 1, ERRID.ERR_ExpectedIdentifier) End If End If @@ -1604,8 +1318,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // The C++ compiler refuses to inline IsIdentifierCharacter, so the ' // < 128 test is inline here. (This loop gets a *lot* of traffic.) ' TODO: make sure we get good perf here - While CanGet(len) - ch = Peek(len) + While Peep(len, ch) Dim code = Convert.ToUInt16(ch) If code < 128 AndAlso IsNarrowIdentifierCharacter(code) OrElse @@ -1619,18 +1332,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax 'Check for a type character Dim TypeCharacter As TypeCharacter = TypeCharacter.None - If CanGet(len) Then - ch = Peek(len) + If Peep(len, ch) Then FullWidthRepeat: Select Case ch Case "!"c ' // If the ! is followed by an identifier it is a dictionary lookup operator, not a type character. - If CanGet(len + 1) Then - Dim NextChar As Char = Peek(len + 1) - - If IsIdentifierStartCharacter(NextChar) OrElse - MatchOneOrAnotherOrFullwidth(NextChar, "["c, "]"c) Then + If Peep(len + 1, cx) Then + If IsIdentifierStartCharacter(cx) OrElse + MatchOneOrAnotherOrFullwidth(cx, "["c, "]"c) Then Exit Select End If End If @@ -1669,9 +1379,7 @@ FullWidthRepeat: Dim contextualKind As SyntaxKind = SyntaxKind.IdentifierToken Dim spelling = GetText(len) - Dim BaseSpelling = If(TypeCharacter = TypeCharacter.None, - spelling, - Intern(spelling, 0, len - 1)) + Dim BaseSpelling = If(TypeCharacter = TypeCharacter.None, spelling, Intern(spelling, 0, len - 1)) ' this can be keyword only if it has no type character, or if it is Mid$ If TypeCharacter = TypeCharacter.None Then @@ -1705,34 +1413,36 @@ FullWidthRepeat: End Function Private Function ScanBracketedIdentifier(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Debug.Assert(CanGet) ' [ - Debug.Assert(Peek() = "["c OrElse Peek() = FULLWIDTH_LEFT_SQUARE_BRACKET) + Dim ch As Char +#If DEBUG Then + Dim ok = Peep(ch) + Debug.Assert(ok) ' [ + Debug.Assert(ch.IsFrom("["c, FULLWIDTH_LEFT_SQUARE_BRACKET)) +#End If Dim IdStart As Integer = 1 Dim Here As Integer = IdStart Dim InvalidIdentifier As Boolean = False - If Not CanGet(Here) Then + If Not Peep(Here, ch) Then Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_MissingEndBrack) End If - Dim ch = Peek(Here) + Dim cx As Char ' check if we can start an ident. If Not IsIdentifierStartCharacter(ch) OrElse (IsConnectorPunctuation(ch) AndAlso - Not (CanGet(Here + 1) AndAlso - IsIdentifierPartCharacter(Peek(Here + 1)))) Then + Not (Peep(Here + 1, cx) AndAlso IsIdentifierPartCharacter(cx))) Then InvalidIdentifier = True End If ' check ident until ] - While CanGet(Here) - Dim [Next] As Char = Peek(Here) + While Peep(Here, cx) - If [Next] = "]"c OrElse [Next] = FULLWIDTH_RIGHT_SQUARE_BRACKET Then + If cx.IsFrom("]"c, FULLWIDTH_RIGHT_SQUARE_BRACKET) Then Dim IdStringLength As Integer = Here - IdStart If IdStringLength > 0 AndAlso Not InvalidIdentifier Then @@ -1754,9 +1464,9 @@ FullWidthRepeat: ' // The sequence "[]" does not define a valid identifier. Return MakeBadToken(precedingTrivia, Here + 1, ERRID.ERR_ExpectedIdentifier) End If - ElseIf IsNewLine([Next]) Then + ElseIf IsNewLine(cx) Then Exit While - ElseIf Not IsIdentifierPartCharacter([Next]) Then + ElseIf Not IsIdentifierPartCharacter(cx) Then InvalidIdentifier = True Exit While End If @@ -1778,7 +1488,9 @@ FullWidthRepeat: End Enum Private Function ScanNumericLiteral(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Debug.Assert(CanGet) + Dim ch As Char + Dim ok = Peep(ch) + Debug.Assert(ok) Dim Here As Integer = 0 Dim IntegerLiteralStart As Integer @@ -1792,10 +1504,9 @@ FullWidthRepeat: ' // First read a leading base specifier, if present, followed by a sequence of zero ' // or more digits. - Dim ch = Peek() - If ch = "&"c OrElse ch = FULLWIDTH_AMPERSAND Then + If ch.IsFrom("&"c, FULLWIDTH_AMPERSAND) Then Here += 1 - ch = If(CanGet(Here), Peek(Here), ChrW(0)) + If Not Peep(Here, ch) Then ch = ChrW(0) FullWidthRepeat: Select Case ch @@ -1804,11 +1515,7 @@ FullWidthRepeat: IntegerLiteralStart = Here Base = LiteralBase.Hexadecimal - While CanGet(Here) - ch = Peek(Here) - If Not IsHexDigit(ch) Then - Exit While - End If + While Peep(Here, ch) AndAlso IsHexDigit(ch) Here += 1 End While @@ -1817,11 +1524,7 @@ FullWidthRepeat: IntegerLiteralStart = Here Base = LiteralBase.Octal - While CanGet(Here) - ch = Peek(Here) - If Not IsOctalDigit(ch) Then - Exit While - End If + While Peep(Here, ch) AndAlso IsOctalDigit(ch) Here += 1 End While @@ -1836,11 +1539,7 @@ FullWidthRepeat: Else ' no base specifier - just go through decimal digits. IntegerLiteralStart = Here - While CanGet(Here) - ch = Peek(Here) - If Not IsDecimalDigit(ch) Then - Exit While - End If + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While End If @@ -1850,21 +1549,16 @@ FullWidthRepeat: ' // Unless there was an explicit base specifier (which indicates an integer literal), ' // read the rest of a float literal. - If Base = LiteralBase.Decimal AndAlso CanGet(Here) Then + If Base = LiteralBase.Decimal AndAlso Peep(Here, ch) Then ' // First read a '.' followed by a sequence of one or more digits. - ch = Peek(Here) - If (ch = "."c Or ch = FULLWIDTH_FULL_STOP) AndAlso - CanGet(Here + 1) AndAlso - IsDecimalDigit(Peek(Here + 1)) Then + Dim cx As Char + If ch.IsFrom("."c, FULLWIDTH_FULL_STOP) AndAlso + (Peep(Here + 1, cx) AndAlso IsDecimalDigit(cx)) Then Here += 2 ' skip dot and first digit ' all following decimal digits belong to the literal (fractional part) - While CanGet(Here) - ch = Peek(Here) - If Not IsDecimalDigit(ch) Then - Exit While - End If + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While literalKind = NumericLiteralKind.Float @@ -1872,24 +1566,16 @@ FullWidthRepeat: ' // Read an exponent symbol followed by an optional sign and a sequence of ' // one or more digits. - If CanGet(Here) AndAlso BeginsExponent(Peek(Here)) Then + If Peep(Here, cx) AndAlso BeginsExponent(cx) Then Here += 1 - If CanGet(Here) Then - ch = Peek(Here) - - If MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then - Here += 1 - End If + If Peep(Here, ch) AndAlso MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then + Here += 1 End If - If CanGet(Here) AndAlso IsDecimalDigit(Peek(Here)) Then + If Peep(Here, cx) AndAlso IsDecimalDigit(cx) Then Here += 1 - While CanGet(Here) - ch = Peek(Here) - If Not IsDecimalDigit(ch) Then - Exit While - End If + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While Else @@ -1908,8 +1594,7 @@ FullWidthRepeat: Dim TypeCharacter As TypeCharacter = TypeCharacter.None - If CanGet(Here) Then - ch = Peek(Here) + If Peep(Here, ch) Then FullWidthRepeat2: Select Case ch @@ -1985,9 +1670,7 @@ FullWidthRepeat2: literalKind = NumericLiteralKind.Decimal ' check if this was not attempt to use obsolete exponent - If CanGet(Here + 1) Then - ch = Peek(Here + 1) - + If Peep(Here + 1, ch) Then If IsDecimalDigit(ch) OrElse MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_ObsoleteExponent) End If @@ -1997,9 +1680,8 @@ FullWidthRepeat2: End If Case "U"c, "u"c - If literalKind <> NumericLiteralKind.Float AndAlso CanGet(Here + 1) Then - Dim NextChar As Char = Peek(Here + 1) - + Dim NextChar As Char + If literalKind <> NumericLiteralKind.Float AndAlso Peep(Here + 1, NextChar) Then 'unsigned suffixes - US, UL, UI If MatchOneOrAnotherOrFullwidth(NextChar, "S"c, "s"c) Then TypeCharacter = TypeCharacter.UShortLiteral @@ -2103,7 +1785,7 @@ FullWidthRepeat2: End If Else - ' // Copy the text of the literal to deal with fullwidth + ' // Copy the text of the literal to deal with fullwidth Dim scratch = GetScratch() For i = 0 To literalWithoutTypeChar - 1 Dim curCh = Peek(i) @@ -2170,31 +1852,17 @@ FullWidthRepeat2: Return Decimal.TryParse(text, NumberStyles.AllowDecimalPoint Or NumberStyles.AllowExponent, CultureInfo.InvariantCulture, value) End Function - Private Function ScanIntLiteral( - ByRef ReturnValue As Integer, - ByRef Here As Integer - ) As Boolean + Private Function ScanIntLiteral(ByRef ReturnValue As Integer, ByRef Here As Integer) As Boolean Debug.Assert(Here >= 0) - - If Not CanGet(Here) Then - Return False - End If - - Dim ch = Peek(Here) - If Not IsDecimalDigit(ch) Then + Dim ch As Char + If Not Peep(Here, ch) OrElse Not IsDecimalDigit(ch) Then Return False End If Dim IntegralValue As Integer = IntegralLiteralCharacterValue(ch) Here += 1 - While CanGet(Here) - ch = Peek(Here) - - If Not IsDecimalDigit(ch) Then - Exit While - End If - + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Dim nextDigit = IntegralLiteralCharacterValue(ch) If IntegralValue < 214748364 OrElse (IntegralValue = 214748364 AndAlso nextDigit < 8) Then @@ -2239,19 +1907,19 @@ FullWidthRepeat2: ' // The first thing has to be an integer, although it's not clear what it is yet If Not ScanIntLiteral(FirstValue, Here) Then Return Nothing - End If ' // If we see a /, then it's a date + Dim c As Char - If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then + If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then Dim FirstDateSeparator As Integer = Here ' // We've got a date HaveDateValue = True Here += 1 - ' Is the first value a year? + ' Is the first value a year? ' It is a year if it consists of exactly 4 digits. ' Condition below uses 5 because we already skipped the separator. If Here - FirstValueStart = 5 Then @@ -2264,10 +1932,10 @@ FullWidthRepeat2: End If ' Do we have a day value? - If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then + If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then ' // Check to see they used a consistent separator - If Peek(Here) <> Peek(FirstDateSeparator) Then + If c <> Peek(FirstDateSeparator) Then GoTo baddate End If @@ -2289,11 +1957,10 @@ FullWidthRepeat2: End If ' // Do we have a year value? - - If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then + If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then ' // Check to see they used a consistent separator - If Peek(Here) <> Peek(FirstDateSeparator) Then + If c <> Peek(FirstDateSeparator) Then GoTo baddate End If @@ -2332,8 +1999,7 @@ FullWidthRepeat2: If HaveTimeValue Then ' // Do we see a :? - - If CanGet(Here) AndAlso IsColon(Peek(Here)) Then + If Peep(Here, c) AndAlso IsColon(c) Then Here += 1 ' // Now let's get the minute value @@ -2345,8 +2011,7 @@ FullWidthRepeat2: HaveMinuteValue = True ' // Do we have a second value? - - If CanGet(Here) AndAlso IsColon(Peek(Here)) Then + If Peep(Here, c) AndAlso IsColon(c) Then ' // Yes. HaveSecondValue = True Here += 1 @@ -2361,24 +2026,21 @@ FullWidthRepeat2: ' // Check AM/PM - If CanGet(Here) Then - If Peek(Here) = "A"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_A OrElse - Peek(Here) = "a"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_A Then - + If Peep(Here, c) Then + If c.IsFrom("A"c, FULLWIDTH_LATIN_CAPITAL_LETTER_A, + "a"c, FULLWIDTH_LATIN_SMALL_LETTER_A) Then HaveAM = True Here += 1 - ElseIf Peek(Here) = "P"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_P OrElse - Peek(Here) = "p"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_P Then - + ElseIf c.IsFrom("P"c, FULLWIDTH_LATIN_CAPITAL_LETTER_P, + "p"c, FULLWIDTH_LATIN_SMALL_LETTER_P) Then HavePM = True Here += 1 End If - - If CanGet(Here) AndAlso (HaveAM OrElse HavePM) Then - If Peek(Here) = "M"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_M OrElse - Peek(Here) = "m"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_M Then + If Peep(Here, c) AndAlso (HaveAM OrElse HavePM) Then + If c.IsFrom("M"c, FULLWIDTH_LATIN_CAPITAL_LETTER_M, + "m"c, FULLWIDTH_LATIN_SMALL_LETTER_M) Then Here = GetWhitespaceLength(Here + 1) @@ -2395,7 +2057,7 @@ FullWidthRepeat2: End If End If - If Not CanGet(Here) OrElse Not IsHash(Peek(Here)) Then + If Not Peep(Here, c) OrElse Not IsHash(c) Then GoTo baddate End If @@ -2500,16 +2162,11 @@ FullWidthRepeat2: baddate: ' // If we can find a closing #, then assume it's a malformed date, ' // otherwise, it's not a date - - While CanGet(Here) - Dim ch As Char = Peek(Here) - If IsHash(ch) OrElse IsNewLine(ch) Then - Exit While - End If + While Peep(Here, c) AndAlso Not (IsHash(c) OrElse IsNewLine(c)) Here += 1 End While - If Not CanGet(Here) OrElse IsNewLine(Peek(Here)) Then + If Not Peep(Here, c) OrElse IsNewLine(c) Then ' // No closing # Return Nothing Else @@ -2530,38 +2187,29 @@ baddate: ' // Check for a Char literal, which can be of the form: ' // """"c or ""c - If CanGet(3) AndAlso IsDoubleQuote(Peek(2)) Then + If Peep(3, ch) AndAlso IsDoubleQuote(Peek(2)) Then If IsDoubleQuote(Peek(1)) Then - If IsDoubleQuote(Peek(3)) AndAlso - CanGet(4) AndAlso - IsLetterC(Peek(4)) Then - + If IsDoubleQuote(ch) AndAlso Peep(4, ch) AndAlso IsLetterC(ch) Then ' // Double-quote Char literal: """"c Return MakeCharacterLiteralToken(precedingTrivia, """"c, 5) End If - ElseIf IsLetterC(Peek(3)) Then + ElseIf IsLetterC(ch) Then ' // Char literal. "x"c Return MakeCharacterLiteralToken(precedingTrivia, Peek(1), 4) End If End If - If CanGet(2) AndAlso - IsDoubleQuote(Peek(1)) AndAlso - IsLetterC(Peek(2)) Then - + If Peep(2, ch) AndAlso IsDoubleQuote(Peek(1)) AndAlso IsLetterC(ch) Then ' // Error. ""c is not a legal char constant Return MakeBadToken(precedingTrivia, 3, ERRID.ERR_IllegalCharConstant) End If Dim scratch = GetScratch() - While CanGet(length) - ch = Peek(length) + While Peep(length, ch) If IsDoubleQuote(ch) Then - If CanGet(length + 1) Then - ch = Peek(length + 1) - + If Peep(length + 1, ch) Then If IsDoubleQuote(ch) Then ' // An escaped double quote scratch.Append(""""c) @@ -2571,7 +2219,6 @@ baddate: ' // The end of the char literal. If IsLetterC(ch) Then ' // Error. "aad"c is not a legal char constant - ' // +2 to include both " and c in the token span scratch.Clear() Return MakeBadToken(precedingTrivia, length + 2, ERRID.ERR_IllegalCharConstant) @@ -2598,7 +2245,7 @@ baddate: ' CC has trouble to prove this after the loop Debug.Assert(CanGet(length - 1)) - '// The literal does not have an explicit termination. + '// The literal does not have an explicit termination. ' DIFFERENT: here in IDE we used to report string token marked as unterminated Dim sp = GetTextNotInterned(length) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index 30e698a182c37..5a8ae75621b38 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -1,9 +1,10 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- +Imports System.Runtime.InteropServices Imports System.Text Imports Microsoft.CodeAnalysis.Text Imports Microsoft.CodeAnalysis.VisualBasic.Symbols @@ -95,6 +96,57 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return p End Function + ''' + ''' Peep at the index position, if it is reachable pass the char back + ''' + ''' How far in front do what to try "peeking" at? + ''' (Passback) the character at that location if possible. + ''' If not the contents are left unaffected. + ''' NOTE: This is different behaviour to the Try methods that always return a value. + ''' + ''' Returns True if it possible to peek at, otherwise returns False. + Friend Function Peep(at As Integer, ByRef ch As Char) As Boolean + ' CanGet(at) + Debug.Assert(_lineBufferOffset + at >= 0) + Debug.Assert(at >= -MaxCharsLookBehind) + If _lineBufferOffset + at >= _bufferLen Then + Return False + End If + ' Peek(at) + at += _lineBufferOffset + Dim page = _curPage + ch = page._arr(at And s_PAGE_MASK) + If page._pageStart <> (at And s_NOT_PAGE_MASK) Then + page = GetPage(at) + ch = page._arr(at And s_PAGE_MASK) + End If + Return True + End Function + + ''' + ''' Peep at the next index position, if it is reachable pass the char back + ''' + ''' (Passback) the character at that location if possible. + ''' If not the contents are left unaffected. + ''' NOTE: This is different behaviour to the Try methods that always return a value. + ''' + ''' Returns True if it possible to peek at, otherwise returns False. + Friend Function Peep(ByRef ch As Char) As Boolean + ' CanGet() + If _lineBufferOffset >= _bufferLen Then + Return False + End If + ' Peek() + Dim page = _curPage + Dim position = _lineBufferOffset + ch = page._arr(position And s_PAGE_MASK) + If page._pageStart <> (position And s_NOT_PAGE_MASK) Then + page = GetPage(position) + ch = page._arr(position And s_PAGE_MASK) + End If + Return True + End Function + ' PERF CRITICAL Private Function Peek(skip As Integer) As Char Debug.Assert(CanGet(skip)) @@ -214,5 +266,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Return result End Function + End Class + End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb index 40a522c7dbfdd..075c1e4ad674e 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Imports Microsoft.CodeAnalysis.VisualBasic.SyntaxFacts @@ -19,15 +19,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim leadingTriviaLength = GetWhitespaceLength(0) Dim offset = leadingTriviaLength Dim length As Integer - - If Not CanGet(offset) Then + Dim c As Char + If Not Peep(offset, c) Then Return MakeEndOfInterpolatedStringToken() End If - Dim c = Peek(offset) - ' This should only ever happen for $" or } - Debug.Assert(leadingTriviaLength = 0 OrElse c = "$"c OrElse c = FULLWIDTH_DOLLAR_SIGN OrElse IsRightCurlyBracket(c)) + Debug.Assert(leadingTriviaLength = 0 OrElse c.IsFrom("$"c, FULLWIDTH_DOLLAR_SIGN) OrElse IsRightCurlyBracket(c)) ' Another } may follow the close brace of an interpolation if the interpolation lacked a format clause. ' This is because the normal escaping rules only apply when parsing the format string. @@ -38,7 +36,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Select Case c Case "$"c, FULLWIDTH_DOLLAR_SIGN - If CanGet(offset + 1) AndAlso IsDoubleQuote(Peek(offset + 1)) Then + If Peep(offset + 1, c) AndAlso IsDoubleQuote(c) Then kind = SyntaxKind.DollarSignDoubleQuoteToken length = 2 scanTrailingTrivia = False ' Trailing whitespace should be scanned as interpolated string text. @@ -112,22 +110,20 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function IsInterpolatedStringPunctuation(Optional offset As Integer = 0) As Boolean - If Not CanGet(offset) Then Return False - - Dim c = Peek(offset) - + Dim c As Char + If Not Peep(offset, c) Then Return False If IsLeftCurlyBracket(c) Then - Return Not CanGet(offset + 1) OrElse Not IsLeftCurlyBracket(Peek(offset + 1)) + Return Not Peep(offset + 1, c) OrElse Not IsLeftCurlyBracket(c) ElseIf IsRightCurlyBracket(c) Then - Return Not CanGet(offset + 1) OrElse Not IsRightCurlyBracket(Peek(offset + 1)) + Return Not Peep(offset + 1, c) OrElse Not IsRightCurlyBracket(c) ElseIf IsDoubleQuote(c) 'A subtle difference between this case and the one above. ' In both interpolated and literal strings the two quote characters used in an escape sequence don't have to match. ' It's enough that the next character is *a* quote char. It doesn't have to be the same quote. ' If we want to preserve consistency the quotes need to be special cased. - Return Not CanGet(offset + 1) OrElse Not IsDoubleQuote(Peek(offset + 1)) + Return Not Peep(offset + 1, c) OrElse Not IsDoubleQuote(c) Else Return False @@ -140,17 +136,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim offset = 0 Dim pendingWhitespace = 0 Dim valueBuilder = GetScratch() - - Do While CanGet(offset) - - Dim c = Peek(offset) + Dim c As Char + Do While Peep(offset, c) ' Any combination of fullwidth and ASCII curly braces of the same direction is an escaping sequence for the corresponding ASCII curly brace. ' We insert that curly brace doubled and because this is the escaping sequence understood by String.Format, that will be replaced by a single brace. ' This is deliberate design and it aligns with existing rules for double quote escaping in strings. If IsLeftCurlyBracket(c) Then - - If CanGet(offset + 1) AndAlso IsLeftCurlyBracket(Peek(offset + 1)) Then + If Peep(offset + 1, c) AndAlso IsLeftCurlyBracket(c) Then ' This is an escape sequence. valueBuilder.Append("{{") @@ -163,8 +156,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do ElseIf IsRightCurlyBracket(c) Then - - If CanGet(offset + 1) AndAlso IsRightCurlyBracket(Peek(offset + 1)) Then + If Peep(offset + 1, c) AndAlso IsRightCurlyBracket(c) Then ' This is an escape sequence. valueBuilder.Append("}}") @@ -177,8 +169,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do ElseIf IsDoubleQuote(c) - - If CanGet(offset + 1) AndAlso IsDoubleQuote(Peek(offset + 1)) Then + If Peep(offset + 1, c) AndAlso IsDoubleQuote(c) Then ' This is a VB double quote escape. Oddly enough this logic allows mixing and matching of ' smart and dumb double quotes in any order. Regardless we always emit as a standard double quote. ' This is consistent with their handling in string literals. diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb index 5e6e421df0beb..a00782ff946f3 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary Option Strict On @@ -16,15 +16,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanXmlTrivia(c As Char) As SyntaxList(Of VisualBasicSyntaxNode) Debug.Assert(Not IsScanningXmlDoc) - Debug.Assert(c = CARRIAGE_RETURN OrElse c = LINE_FEED OrElse c = " "c OrElse c = CHARACTER_TABULATION) + Debug.Assert(c.IsFrom(CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION)) Dim builder = _triviaListPool.Allocate Dim len = 0 Do - If c = " "c OrElse c = CHARACTER_TABULATION Then + If c.IsFrom(" "c, CHARACTER_TABULATION) Then len += 1 - ElseIf c = CARRIAGE_RETURN OrElse c = LINE_FEED Then + ElseIf c.IsFrom(CARRIAGE_RETURN, LINE_FEED) Then If len > 0 Then builder.Add(MakeWhiteSpaceTrivia(GetText(len))) len = 0 @@ -34,10 +34,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do End If - If Not CanGet(len) Then + If Not Peep(len, c) Then Exit Do End If - c = Peek(len) Loop If len > 0 Then @@ -69,8 +68,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing - While CanGet() - Dim c As Char = Peek() + Dim c As Char + While Peep(c) Select Case (c) ' // Whitespace @@ -93,7 +92,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax leadingTrivia = ScanXmlTrivia(c) Case "/"c - If CanGet(1) AndAlso Peek(1) = ">" Then + If NextIs(1, ">"c) Then Return XmlMakeEndEmptyElementToken(leadingTrivia) End If Return XmlMakeDivToken(leadingTrivia) @@ -113,29 +112,30 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeDoubleQuoteToken(leadingTrivia, c, isOpening:=True) Case "<"c - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If Peep(1, ch) Then + Dim c2 As Char Select Case ch Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + If Peep(2, c2) Then + Select Case c2 Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(leadingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(leadingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If Nextare(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(leadingTrivia) End If End Select End If Return XmlLessThanExclamationToken(state, leadingTrivia) Case "%"c - If NextIs(2,"="c) Then + If NextIs(2, "="c) Then Return XmlMakeBeginEmbeddedToken(leadingTrivia) End If Case "?"c @@ -148,37 +148,36 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeLessToken(leadingTrivia) Case "?"c - - If NextIs(1,">"c) Then + If NextIs(1, ">"c) Then ' // Create token for the '?>' termination sequence Return XmlMakeEndProcessingInstructionToken(leadingTrivia) - End If + End If - Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) + Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) - Case "("c - Return XmlMakeLeftParenToken(leadingTrivia) + Case "("c + Return XmlMakeLeftParenToken(leadingTrivia) - Case ")"c - Return XmlMakeRightParenToken(leadingTrivia) + Case ")"c + Return XmlMakeRightParenToken(leadingTrivia) - Case "!"c, ";"c, "#"c, ","c, "}"c - Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) + Case "!"c, ";"c, "#"c, ","c, "}"c + Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) - Case ":"c - Return XmlMakeColonToken(leadingTrivia) + Case ":"c + Return XmlMakeColonToken(leadingTrivia) - Case "["c - Return XmlMakeOpenBracketToken(state, leadingTrivia) + Case "["c + Return XmlMakeOpenBracketToken(state, leadingTrivia) - Case "]"c - Return XmlMakeCloseBracketToken(state, leadingTrivia) + Case "]"c + Return XmlMakeCloseBracketToken(state, leadingTrivia) - Case Else - ' // Because of weak scanning of QName, this state must always handle - ' // '=' | '\'' | '"'| '/' | '>' | '<' | '?' + Case Else + ' // Because of weak scanning of QName, this state must always handle + ' // '=' | '\'' | '"'| '/' | '>' | '<' | '?' - Return ScanXmlNcName(leadingTrivia) + Return ScanXmlNcName(leadingTrivia) End Select End While @@ -186,7 +185,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function '// - '// This is used to detect a VB statement on the next line + '// This is used to detect a VB statement on the next line '// '// NL WS* KW WS* ID | KW '// Example Dim x @@ -200,21 +199,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '// '// NL WS* < ID WS* ( '// Example SyntaxKind.XmlNameToken Then leadingTrivia = ScanSingleLineTrivia() c = Peek() - possibleStatement = - c = "("c OrElse c = FULLWIDTH_LEFT_PARENTHESIS + possibleStatement = c.IsFrom("("c, FULLWIDTH_LEFT_PARENTHESIS) End If End If @@ -309,8 +307,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim scratch = GetScratch() - While CanGet(Here) - Dim c As Char = Peek(Here) + Dim c As Char + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -336,28 +334,29 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeTextLiteralToken(Nothing, Here, scratch) Else scratch.Clear() ' will not use this - Here = 0 ' consumed chars. + Here = 0 ' consumed chars. precedingTrivia = ScanXmlTrivia(Peek) End If End If Debug.Assert(Here = 0) - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If Peep(1, ch) Then Select Case ch Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + Dim c2 As Char + If Peep(2, c2) Then + Select Case c2 Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If NextAre(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If End Select @@ -372,102 +371,100 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeBeginEndElementToken(precedingTrivia, s_scanNoTriviaFunc) End Select End If - Return XmlMakeLessToken(precedingTrivia) Case "]"c - If NextAre(Here+1,"]>") Then - + If NextAre(Here + 1, "]>") Then ' // If valid characters found then return them. If Here <> 0 Then - Return XmlMakeTextLiteralToken(Nothing, Here, scratch) - End If - - ' // Create an invalid character data token for the illegal ']]>' sequence - Return XmlMakeTextLiteralToken(Nothing, 3, ERRID.ERR_XmlEndCDataNotAllowedInContent) - End If - GoTo ScanChars - - Case "#"c - ' // Even though # is valid in content, abort xml scanning if the m_State shows and error - ' // and the line begins with NL WS* # WS* KW - - 'TODO: error recovery - how can we do ths? - - 'If m_State.m_IsXmlError Then - ' MakeXmlCharToken(tokens.tkXmlCharData, Here - m_InputStreamPosition, IsAllWhitespace) - ' m_InputStreamPosition = Here - - ' Dim sharp As Token = MakeToken(tokens.tkSharp, 1) - ' m_InputStreamPosition += 1 - - ' While (m_InputStream(m_InputStreamPosition) = " "c OrElse m_InputStream(m_InputStreamPosition) = CHARACTER_TABULATION) - ' m_InputStreamPosition += 1 - ' End While - - ' ScanXmlQName() - - ' Dim restart As Token = CheckXmlForStatement() - - ' If restart IsNot Nothing Then - ' ' // Abort Xml - Found Keyword space at the beginning of the line - ' AbandonTokens(restart) - ' m_State.Init(LexicalState.VB) - ' MakeToken(tokens.tkXmlAbort, 0) - ' Return - ' End If - - ' AbandonTokens(sharp) - ' Here = m_InputStreamPosition - 'End If - GoTo ScanChars + Return XmlMakeTextLiteralToken(Nothing, Here, scratch) + End If - Case "%"c - - 'TODO: error recovery. We cannot do this. - 'If there is all whitespace after ">", it will be scanned as insignificant, - 'but in this case it is significant. - 'Also as far as I can see Dev10 does not resync on "%>" text anyways. - - '' // Even though %> is valid in pcdata. When inside of an embedded expression - '' // return this sequence separately so that the xml literal completion code can - '' // easily detect the end of an embedded expression that may be temporarily hidden - '' // by a new element. i.e. <%= %> - - 'If CanGetCharAtOffset(Here + 1) AndAlso _ - ' PeekAheadChar(Here + 1) = ">"c Then - - ' ' // If valid characters found then return them. - ' If Here <> 0 Then - ' Return XmlMakeCharDataToken(Nothing, Here, New String(value.ToArray)) - ' End If - - ' ' // Create a special pcdata token for the possible tkEndXmlEmbedded - ' Return XmlMakeCharDataToken(Nothing, 2, "%>") - 'Else - ' IsAllWhitespace = False - ' value.Add("%"c) - ' Here += 1 - 'End If - 'Continue While - GoTo ScanChars - Case Else + ' // Create an invalid character data token for the illegal ']]>' sequence + Return XmlMakeTextLiteralToken(Nothing, 3, ERRID.ERR_XmlEndCDataNotAllowedInContent) + End If + GoTo ScanChars + + Case "#"c + ' // Even though # is valid in content, abort xml scanning if the m_State shows and error + ' // and the line begins with NL WS* # WS* KW + + 'TODO: error recovery - how can we do ths? + + 'If m_State.m_IsXmlError Then + ' MakeXmlCharToken(tokens.tkXmlCharData, Here - m_InputStreamPosition, IsAllWhitespace) + ' m_InputStreamPosition = Here + + ' Dim sharp As Token = MakeToken(tokens.tkSharp, 1) + ' m_InputStreamPosition += 1 + + ' While (m_InputStream(m_InputStreamPosition) = " "c OrElse m_InputStream(m_InputStreamPosition) = CHARACTER_TABULATION) + ' m_InputStreamPosition += 1 + ' End While + + ' ScanXmlQName() + + ' Dim restart As Token = CheckXmlForStatement() + + ' If restart IsNot Nothing Then + ' ' // Abort Xml - Found Keyword space at the beginning of the line + ' AbandonTokens(restart) + ' m_State.Init(LexicalState.VB) + ' MakeToken(tokens.tkXmlAbort, 0) + ' Return + ' End If + + ' AbandonTokens(sharp) + ' Here = m_InputStreamPosition + 'End If + GoTo ScanChars + + Case "%"c + + 'TODO: error recovery. We cannot do this. + 'If there is all whitespace after ">", it will be scanned as insignificant, + 'but in this case it is significant. + 'Also as far as I can see Dev10 does not resync on "%>" text anyways. + + '' // Even though %> is valid in pcdata. When inside of an embedded expression + '' // return this sequence separately so that the xml literal completion code can + '' // easily detect the end of an embedded expression that may be temporarily hidden + '' // by a new element. i.e. <%= %> + + 'If CanGetCharAtOffset(Here + 1) AndAlso _ + ' PeekAheadChar(Here + 1) = ">"c Then + + ' ' // If valid characters found then return them. + ' If Here <> 0 Then + ' Return XmlMakeCharDataToken(Nothing, Here, New String(value.ToArray)) + ' End If + + ' ' // Create a special pcdata token for the possible tkEndXmlEmbedded + ' Return XmlMakeCharDataToken(Nothing, 2, "%>") + 'Else + ' IsAllWhitespace = False + ' value.Add("%"c) + ' Here += 1 + 'End If + 'Continue While + GoTo ScanChars + Case Else ScanChars: - ' // Check characters are valid - IsAllWhitespace = False - Dim xmlCh = ScanXmlChar(Here) - - If xmlCh.Length = 0 Then - ' bad char - If Here > 0 Then - Return XmlMakeTextLiteralToken(Nothing, Here, scratch) - Else - Return XmlMakeBadToken(Nothing, 1, ERRID.ERR_IllegalChar) + ' // Check characters are valid + IsAllWhitespace = False + Dim xmlCh = ScanXmlChar(Here) + + If xmlCh.Length = 0 Then + ' bad char + If Here > 0 Then + Return XmlMakeTextLiteralToken(Nothing, Here, scratch) + Else + Return XmlMakeBadToken(Nothing, 1, ERRID.ERR_IllegalChar) + End If End If - End If - xmlCh.AppendTo(scratch) - Here += xmlCh.Length + xmlCh.AppendTo(scratch) + Here += xmlCh.Length End Select End While @@ -492,69 +489,66 @@ ScanChars: End If Dim Here = 0 - While CanGet(Here) - Dim c As Char = Peek(Here) - Select Case (c) + Dim c As Char + While Peep(Here, c) + Select Case c Case CARRIAGE_RETURN, LINE_FEED Return XmlMakeCommentToken(precedingTrivia, Here + LengthOfLineBreak(c, Here)) Case "-"c - If NextIs(Here + 1,"-"c) Then - + If NextIs(Here + 1, "-"c) Then ' // --> terminates an Xml comment but otherwise -- is an illegal character sequence. ' // The scanner will always returns "--" as a separate comment data string and the ' // the semantics will error if '--' is ever found. ' // Return valid characters up to the -- If Here > 0 Then - Return XmlMakeCommentToken(precedingTrivia, Here) - End If - - If CanGet(Here + 2) Then + Return XmlMakeCommentToken(precedingTrivia, Here) + End If - c = Peek(Here + 2) - Here += 2 - ' // if > is not found then this is an error. Return the -- string + If Peep(Here + 2, c) Then + Here += 2 + ' // if > is not found then this is an error. Return the -- string - If c <> ">"c Then - Return XmlMakeCommentToken(precedingTrivia, 2) + If c <> ">"c Then + Return XmlMakeCommentToken(precedingTrivia, 2) - ' TODO: we cannot do the following + ' TODO: we cannot do the following - ' // For better error recovery, allow -> to terminate the comment. - ' // This works because the -> terminates only when the invalid -- - ' // is returned. + ' // For better error recovery, allow -> to terminate the comment. + ' // This works because the -> terminates only when the invalid -- + ' // is returned. - 'If Here + 1 < m_InputStreamEnd AndAlso _ - ' m_InputStream(Here) = "-"c AndAlso _ - ' m_InputStream(Here + 1) = ">"c Then + 'If Here + 1 < m_InputStreamEnd AndAlso _ + ' m_InputStream(Here) = "-"c AndAlso _ + ' m_InputStream(Here + 1) = ">"c Then - ' Here += 1 - 'Else - ' Continue While - 'End If - Else - Return XmlMakeEndCommentToken(precedingTrivia) + ' Here += 1 + 'Else + ' Continue While + 'End If + Else + Return XmlMakeEndCommentToken(precedingTrivia) + End If End If End If - End If - GoTo ScanChars + GoTo ScanChars - Case Else + Case Else ScanChars: - Dim xmlCh = ScanXmlChar(Here) - If xmlCh.Length <> 0 Then - Here += xmlCh.Length - Continue While - End If + Dim xmlCh = ScanXmlChar(Here) + If xmlCh.Length <> 0 Then + Here += xmlCh.Length + Continue While + End If - ' bad char - If Here > 0 Then - Return XmlMakeCommentToken(precedingTrivia, Here) - Else - Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalChar) - End If + ' bad char + If Here > 0 Then + Return XmlMakeCommentToken(precedingTrivia, Here) + Else + Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalChar) + End If End Select End While @@ -583,18 +577,16 @@ ScanChars: Dim scratch = GetScratch() Dim Here = 0 - - While CanGet(Here) - Dim c As Char = Peek(Here) - Select Case (c) - + Dim c As Char + While Peep(Here, c) + Select Case c Case CARRIAGE_RETURN, LINE_FEED Here = SkipLineBreak(c, Here) scratch.Append(LINE_FEED) Return XmlMakeCDataToken(precedingTrivia, Here, scratch) Case "]"c - If NextAre(Here+1,"]>") Then + If NextAre(Here + 1, "]>") Then '// If valid characters found then return them. If Here <> 0 Then @@ -647,11 +639,10 @@ ScanChars: Dim precedingTrivia = _triviaListPool.Allocate(Of VisualBasicSyntaxNode)() Dim result As SyntaxToken - - If state = ScannerState.StartProcessingInstruction AndAlso CanGet() Then + Dim c As Char + If state = ScannerState.StartProcessingInstruction AndAlso Peep(c) Then ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ - Dim c = Peek() Select Case c Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION Dim wsTrivia = ScanXmlTrivia(c) @@ -660,8 +651,7 @@ ScanChars: End If Dim Here = 0 - While CanGet(Here) - Dim c As Char = Peek(Here) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -670,7 +660,6 @@ ScanChars: Case "?"c If NextIs(Here + 1, ">"c) Then - '// If valid characters found then return them. If Here <> 0 Then result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here) @@ -720,9 +709,8 @@ CleanUp: ' // Misc ::= Comment | PI | S Dim precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing - While CanGet() - Dim c As Char = Peek() - + Dim c As Char + While Peep(c) Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -732,17 +720,17 @@ CleanUp: precedingTrivia = ScanXmlTrivia(c) Case "<"c - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If Peep(1, ch) Then Select Case ch Case "!"c - If NextAre(2,"--") Then + If NextAre(2, "--") Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) - ElseIf NextAre(2,"DOCTYPE") Then + ElseIf NextAre(2, "DOCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If Case "%"c - If NextIs(2,"="c) Then + If NextIs(2, "="c) Then Return XmlMakeBeginEmbeddedToken(precedingTrivia) End If Case "?"c @@ -752,7 +740,7 @@ CleanUp: Return XmlMakeLessToken(precedingTrivia) - ' TODO: review + ' TODO: review ' If Not m_State.m_ScannedElement OrElse c = "?"c OrElse c = "!"c Then ' ' // Remove tEOL from token ring if any exists @@ -793,14 +781,12 @@ CleanUp: ' This can never happen as this token cannot cross lines. Debug.Assert(Not (IsScanningXmlDoc AndAlso IsAtNewLine())) + Dim c As Char Dim Here = 0 Dim scratch = GetScratch() - While CanGet(Here) - Dim c As Char = Peek(Here) - + While Peep(Here, c) Select Case (c) - Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION If Here > 0 Then Return XmlMakeAttributeDataToken(Nothing, Here, scratch) @@ -888,9 +874,8 @@ ScanChars: Dim Here = 0 Dim scratch = GetScratch() - - While CanGet(Here) - Dim c As Char = Peek(Here) + Dim c As Char + While Peep(Here, c) If c = terminatingChar Or c = altTerminatingChar Then If Here > 0 Then result = XmlMakeAttributeDataToken(precedingTrivia, Here, scratch) @@ -923,7 +908,7 @@ ScanChars: GoTo CleanUp Else ' report unexpected <%= in a special way. - If NextAre(1,"%=") Then + If NextAre(1, "%=") Then Dim errEmbedStart = XmlMakeAttributeDataToken(precedingTrivia, 3, "<%=") Dim errEmberinfo = ErrorFactory.ErrorInfo(ERRID.ERR_QuotedEmbeddedExpression) result = DirectCast(errEmbedStart.SetDiagnostics({errEmberinfo}), SyntaxToken) @@ -984,17 +969,15 @@ CleanUp: End Function ''' - ''' 0 - not a surrogate, 2 - is valid surrogate + ''' 0 - not a surrogate, 2 - is valid surrogate ''' 1 is an error ''' Private Function ScanSurrogatePair(c1 As Char, Here As Integer) As XmlCharResult Debug.Assert(Here >= 0) Debug.Assert(CanGet(Here)) Debug.Assert(Peek(Here) = c1) - - If IsHighSurrogate(c1) AndAlso CanGet(Here + 1) Then - Dim c2 = Peek(Here + 1) - + Dim c2 As Char + If IsHighSurrogate(c1) AndAlso Peep(Here + 1, c2) Then If IsLowSurrogate(c2) Then Return New XmlCharResult(c1, c2) End If @@ -1003,7 +986,7 @@ CleanUp: Return Nothing End Function - ' contains result of Xml char scanning. + ' contains result of Xml char scanning. Friend Structure XmlCharResult Friend ReadOnly Length As Integer Friend ReadOnly Char1 As Char @@ -1033,9 +1016,9 @@ CleanUp: Private Function ScanXmlChar(Here As Integer) As XmlCharResult Debug.Assert(Here >= 0) - Debug.Assert(CanGet(Here)) - - Dim c = Peek(Here) + Dim c As Char + Dim ok = Peep(Here, c) + Debug.Assert(ok) If Not isValidUtf16(c) Then Return Nothing @@ -1066,9 +1049,8 @@ CleanUp: 'TODO - Fix ScanXmlNCName to conform to XML spec instead of old loose scanning. - While CanGet(Here) - Dim c As Char = Peek(Here) - + Dim c As Char + While Peep(Here, c) Select Case (c) Case ":"c, " "c, CHARACTER_TABULATION, LINE_FEED, CARRIAGE_RETURN, @@ -1130,13 +1112,10 @@ CreateNCNameToken: End Function Private Function ScanXmlReference(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As XmlTextTokenSyntax - Debug.Assert(CanGet) - Debug.Assert(Peek() = "&"c) - + Debug.Assert(NextIs(0, "&"c)) + Dim c, c1 As Char ' skip 1 char for "&" - If CanGet(1) Then - Dim c As Char = Peek(1) - + If Peep(1, c) Then Select Case (c) Case "#"c Dim Here = 2 ' skip "&#" @@ -1150,7 +1129,7 @@ CreateNCNameToken: value = Intern({result.Char1, result.Char2}) End If - If CanGet(Here) AndAlso Peek(Here) = ";"c Then + If NextIs(Here, ";"c) Then Return XmlMakeEntityLiteralToken(precedingTrivia, Here + 1, value) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, Here, value) @@ -1162,9 +1141,8 @@ CreateNCNameToken: Case "a"c ' // & ' // ' - - If CanGet(4) AndAlso NextAre(2,"mp") Then - If Peek(4) = ";"c Then + If Peep(4, c1) AndAlso NextAre(2, "mp") Then + If c1 = ";"c Then Return XmlMakeAmpLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 4, "&") @@ -1172,9 +1150,8 @@ CreateNCNameToken: Return DirectCast(noSemicolon.SetDiagnostics({noSemicolonError}), XmlTextTokenSyntax) End If - ElseIf CanGet(5) AndAlso NextAre(2,"pos") Then - - If Peek(5) = ";"c Then + ElseIf Peep(5, c1) AndAlso NextAre(2, "pos") Then + If c1 = ";"c Then Return XmlMakeAposLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 5, "'") @@ -1185,10 +1162,8 @@ CreateNCNameToken: Case "l"c ' // < - - If CanGet(3) AndAlso NextIs(2,"t"c) Then - - If Peek(3) = ";"c Then + If Peep(3, c1) AndAlso NextIs(2, "t"c) Then + If c1 = ";"c Then Return XmlMakeLtLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 3, "<") @@ -1199,10 +1174,8 @@ CreateNCNameToken: Case "g"c ' // > - - If CanGet(3) AndAlso NextIs(2,"t"c) Then - - If Peek(3) = ";"c Then + If Peep(3, c1) AndAlso NextIs(2, "t"c) Then + If c1 = ";"c Then Return XmlMakeGtLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 3, ">") @@ -1213,10 +1186,8 @@ CreateNCNameToken: Case "q"c ' // " - - If CanGet(5) AndAlso NextAre(2,"uot") Then - - If Peek(5) = ";"c Then + If Peep(5, c1) AndAlso NextAre(2, "uot") Then + If c1 = ";"c Then Return XmlMakeQuotLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 5, """") @@ -1236,21 +1207,18 @@ CreateNCNameToken: Private Function ScanXmlCharRef(ByRef index As Integer) As XmlCharResult Debug.Assert(index >= 0) - - If Not CanGet(index) Then + Dim ch As Char + If Not Peep(index, ch) Then Return Nothing End If ' cannot reuse Scratch as this can be used in a nested call. Dim charRefSb As New StringBuilder Dim Here = index - - Dim ch = Peek(Here) If ch = "x"c Then Here += 1 - While CanGet(Here) - ch = Peek(Here) + While Peep(Here, ch) If XmlCharType.IsHexDigit(ch) Then charRefSb.Append(ch) Else @@ -1266,13 +1234,9 @@ CreateNCNameToken: Return result End If Else - While CanGet(Here) - ch = Peek(Here) - If XmlCharType.IsDigit(ch) Then - charRefSb.Append(ch) - Else - Exit While - End If + + While Peep(Here, ch) AndAlso XmlCharType.IsDigit(ch) + charRefSb.Append(ch) Here += 1 End While If charRefSb.Length > 0 Then @@ -1285,5 +1249,7 @@ CreateNCNameToken: End If Return Nothing End Function + End Class + End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb index 2f402543c7229..6231790bcd4c2 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary diff --git a/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb b/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb index 7080f6ad81ff8..d6af5c85865ec 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary Option Strict On @@ -151,8 +151,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Gets current reusable syntax node. - ''' If node is returned its start will be aligned with the start of current token. - ''' NOTE: Line offset may not match start of current token because of lookahead. + ''' If node is returned its start will be aligned with the start of current token. + ''' NOTE: Line offset may not match start of current token because of lookahead. ''' Friend Overridable Function GetCurrentSyntaxNode() As VisualBasicSyntaxNode Return Nothing @@ -161,12 +161,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Indicates that previously returned node has been consumed ''' and scanner needs to advance by the size of the node. - ''' + ''' ''' NOTE: the advancement is done relative to the start of the current token. - ''' Line offset may not match start of current token because of lookahead. - ''' - ''' This operation will discard lookahead tokens and reset preprocessor state - ''' to the state of current token. + ''' Line offset may not match start of current token because of lookahead. + ''' + ''' This operation will discard lookahead tokens and reset preprocessor state + ''' to the state of current token. ''' Friend Overridable Sub MoveToNextSyntaxNode() ' do not use prev token after consuming a nonterminal @@ -180,14 +180,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Indicates that previously returned node has been consumed ''' and scanner needs to advance by the size of the node. - ''' + ''' ''' NOTE: the advancement is done relative to the _lineBufferOffset. ''' Line offset will likely not match start of current token because this operation ''' is done while constructing the content of current token. - ''' + ''' ''' NOTE: This operation assumes that there is no tokens read ahead. - ''' - ''' NOTE: This operation does not change preprocessor state. + ''' + ''' NOTE: This operation does not change preprocessor state. ''' The assumption is that it is responsibility of the node consumer to update preprocessor ''' state if needed when using nodes that change preprocessor state. ''' @@ -236,7 +236,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If state <> _currentToken.State Then ' this is a special case for switching from VB to Xml - ' we need to keep preceding trivia as it was scanned + ' we need to keep preceding trivia as it was scanned If _currentToken.State = ScannerState.VB AndAlso state = ScannerState.Content Then Dim vbTk = GetCurrentToken() diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index 175b5145f172f..54af92bbe03e4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -49,9 +49,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function TryScanXmlDocComment(tList As SyntaxListBuilder) As Boolean Debug.Assert(IsAtNewLine) - ' leading whitespace until we see ''' should be regular whitespace - If CanGet() AndAlso IsWhitespace(Peek()) Then + Dim c As Char + If Peep(c) AndAlso IsWhitespace(c) Then Dim ws = ScanWhitespace() tList.Add(ws) End If @@ -62,8 +62,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #End If Dim restorePoint = CreateRestorePoint() - ' since we do not have lookahead tokens, this just - ' resets current token to _lineBufferOffset + ' since we do not have lookahead tokens, this just + ' resets current token to _lineBufferOffset Me.GetNextTokenInState(ScannerState.Content) Dim currentNonterminal = Me.GetCurrentSyntaxNode() @@ -79,23 +79,23 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Me.IsScanningXmlDoc = True Me._isStartingFirstXmlDocLine = True - ' NOTE: Documentation comment syntax trivia must have at least one child xml node, because + ' NOTE: Documentation comment syntax trivia must have at least one child xml node, because ' all the ['''] trivia are created as leading trivia for appropriate tokens. ' This means that we have to create at least one XmlText having trailing ' EOL to represent an empty documentation comment: ['''] ' ' The problem with this approach is that in presence of some errors (like ' not closed XML tags) we create missing tokens needed to represent the nodes - ' *after* that last of the doc comment trivia, that means all the locations + ' *after* that last of the doc comment trivia, that means all the locations ' of created diagnostics will land on the first character of the next line ' after documentation comment ' - ' To workaround this we parse XML nodes in two phases: - ' - in the first phase we detect the last DocCommentLineBreak and create - ' end-of-xml token instead; this should force all diagnostics to be + ' To workaround this we parse XML nodes in two phases: + ' - in the first phase we detect the last DocCommentLineBreak and create + ' end-of-xml token instead; this should force all diagnostics to be ' reported on the next token location; - ' - in the second phase we continue parsing XML nodes but don't create - ' end-of-xml token which should just result in parsing one single node + ' - in the second phase we continue parsing XML nodes but don't create + ' end-of-xml token which should just result in parsing one single node ' of XmlText type containing EOL; ' Then we merge the results and create resulting DocumentationCommentTrivia @@ -106,7 +106,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' The second phase Me._endOfXmlInsteadOfLastDocCommentLineBreak = False If nodes.Count = 0 AndAlso parser.CurrentToken.Kind = SyntaxKind.EndOfXmlToken Then - ' This must be an empty documentation comment, we need to reset scanner so + ' This must be an empty documentation comment, we need to reset scanner so ' that the doc comment exterior trivia ([''']) lands on the final XmlNode ResetLineBufferOffset() @@ -147,14 +147,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' lexes (ws)''' Private Function TrySkipXmlDocMarker(ByRef len As Integer) As Boolean + Dim c As Char Dim Here = len - While CanGet(Here) - Dim c = Peek(Here) - If IsWhitespace(c) Then - Here += 1 - Else - Exit While - End If + While Peep(Here, c) AndAlso IsWhitespace(c) + Here += 1 End While If StartsXmlDoc(Here) Then @@ -186,7 +182,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim len = 0 Do - If c = " "c OrElse c = CHARACTER_TABULATION Then + If c.IsFrom(" "c, CHARACTER_TABULATION) Then len += 1 ElseIf IsNewLine(c) Then @@ -240,9 +236,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim Here As Integer = 0 Dim scratch = GetScratch() - - While CanGet(Here) - Dim c As Char = Peek(Here) + Dim c As Char + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -255,7 +250,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If _endOfXmlInsteadOfLastDocCommentLineBreak Then Dim tempHere As Integer = Here If Not TrySkipXmlDocMarker(tempHere) Then - ' NOTE: we need to reset the buffer so that precedingTrivia + ' NOTE: we need to reset the buffer so that precedingTrivia ' lands on the next token ResetLineBufferOffset() Return SyntaxFactory.Token(Nothing, SyntaxKind.EndOfXmlToken, Nothing, String.Empty) @@ -282,22 +277,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Debug.Assert(Here = 0) - If CanGet(1) Then - Dim ch As Char = Peek(1) - Select Case ch + If Peep(1, c) Then + Select Case c Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + If Peep(2, c) Then + Select Case c Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If NextAre(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If End Select @@ -325,7 +319,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax GoTo ScanChars Case Else ScanChars: - ' // Check characters are valid + ' // Check characters are valid Dim xmlCh = ScanXmlChar(Here) If xmlCh.Length = 0 Then @@ -370,10 +364,10 @@ ScanChars: precedingTrivia.Add(xDocTrivia) End If - If state = ScannerState.StartProcessingInstruction AndAlso CanGet() Then + Dim c As Char + If state = ScannerState.StartProcessingInstruction AndAlso Peep(c) Then ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ - Dim c = Peek() Select Case c Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION Dim offsets = CreateOffsetRestorePoint() @@ -387,8 +381,7 @@ ScanChars: End If Dim Here = 0 - While CanGet(Here) - Dim c As Char = Peek(Here) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -396,8 +389,7 @@ ScanChars: GoTo CleanUp Case "?"c - If NextIs(Here + 1,">"c) Then - + If NextIs(Here + 1, ">"c) Then '// If valid characters found then return them. If Here <> 0 Then result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here) @@ -461,7 +453,8 @@ CleanUp: precedingTrivia = New SyntaxList(Of VisualBasicSyntaxNode)(xDocTrivia) End If - While CanGet() + Dim c As Char + While Peep(c) If Not precedingTrivia.Any AndAlso IsAtNewLine() AndAlso Not Me._doNotRequireXmlDocCommentPrefix Then ' this would indicate that we looked at Trivia, but did not find ' XmlDoc prefix (or we would not be at the line start) @@ -469,8 +462,6 @@ CleanUp: Return MakeEofToken(precedingTrivia) End If - Dim c As Char = Peek() - Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -506,22 +497,21 @@ CleanUp: Return XmlMakeDoubleQuoteToken(precedingTrivia, c, isOpening:=True) Case "<"c - If CanGet(1) Then - Dim ch As Char = Peek(1) - Select Case ch + If Peep(1, c) Then + Select Case c Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + If Peep(2, c) Then + Select Case c Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If NextAre(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If End Select @@ -550,11 +540,7 @@ CleanUp: Case ")"c Return XmlMakeRightParenToken(precedingTrivia) - Case "!"c, - ";"c, - "#"c, - ","c, - "}"c + Case "!"c, ";"c, "#"c, ","c, "}"c Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) Case ":"c diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb index 5ed11a1fcb293..fc5767cffcf6d 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary @@ -298,7 +298,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function XmlMakeBeginEndElementToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax Debug.Assert(NextAre("")) - AdvanceChar(2) Return MakePunctuationToken(SyntaxKind.SlashGreaterThanToken, "/>", precedingTrivia, Nothing) End Function @@ -319,11 +317,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function XmlMakeEndEmbeddedToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax - Debug.Assert(Peek() = "%"c OrElse Peek() = FULLWIDTH_PERCENT_SIGN) + Dim c = Peek + Debug.Assert(c.IsFrom("%"c, FULLWIDTH_PERCENT_SIGN)) Debug.Assert(Peek(1) = ">"c) Dim spelling As String - If Peek() = "%"c Then + If c = "%"c Then AdvanceChar(2) spelling = "%>" Else @@ -354,7 +353,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function XmlMakeCloseBracketToken(state As ScannerState, precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As BadTokenSyntax Debug.Assert(Peek() = "]"c) - Return XmlMakeBadToken(SyntaxSubKind.CloseBracketToken, precedingTrivia, 1, If(state = ScannerState.DocType, ERRID.ERR_DTDNotSupported, ERRID.ERR_IllegalXmlNameChar)) End Function #End Region @@ -413,7 +411,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #Region "CData" Private Function XmlMakeBeginCDataToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax Debug.Assert(NextAre(" SyntaxKind.None Then - Return String.Format("[{0}]", identifier) + Return $"[{identifier}]" End If ' The minimal flag is e.g. used by the service layer when generating code (example: simplify name). @@ -113,13 +113,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic If Me.IsMinimizing Then Dim contextualKeywordKind As SyntaxKind = SyntaxFacts.GetContextualKeywordKind(identifier) - ' Leading implicit line continuation is allowed before query operators (Aggregate, Distinct, From, Group By, - ' Group Join, Join, Let, Order By, Select, Skip, Skip While, Take, Take While, Where, In, Into, On, Ascending, + ' Leading implicit line continuation is allowed before query operators (Aggregate, Distinct, From, Group By, + ' Group Join, Join, Let, Order By, Select, Skip, Skip While, Take, Take While, Where, In, Into, On, Ascending, ' and Descending). ' If the current identifier is one of these keywords, we need to escape them to avoid parsing issues if the ' previous line was a query expression. ' - ' In addition to the query operators, we need to escape the identifier "preserve" to avoid ambiguities + ' In addition to the query operators, we need to escape the identifier "preserve" to avoid ambiguities ' inside of a dim statement (dim [preserve].ArrayName(1)) Select Case contextualKeywordKind Case SyntaxKind.AggregateKeyword, @@ -139,7 +139,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic SyntaxKind.AscendingKeyword, SyntaxKind.DescendingKeyword, SyntaxKind.PreserveKeyword - Return String.Format("[{0}]", identifier) + Return $"[{identifier}]" End Select End If diff --git a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb index 0d6f65a80c7f4..2bcd96e563d3f 100644 --- a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb +++ b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb @@ -16,7 +16,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Debug.Assert(symbol.ContainingNamespace IsNot Nothing OrElse symbol.IsGlobalNamespace) ' NOTE(cyrusn): We only call this once we've already checked if there is an alias that - ' corresponds to this namespace. + ' corresponds to this namespace. If symbol.IsGlobalNamespace Then ' nothing to add for global namespace itself