From f63a208e4d1935fd4247b96f0cf03746f9fb4c0e Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Thu, 26 Mar 2015 20:28:17 +0000 Subject: [PATCH 1/9] [VB: Scanner] Unifies 2 scanner methods + Rename Commonly Used Methods Unification =========== The methods `TryScanToken` and `ScanTokenFullWidth` have extremely similar functionality. So I've unified the content of both into 'ScanTokenCommon'. The existing methods are retained (with internal changes to refer to the common method) to maintain compatibility with calls to those exist methods. Renamed Methods =============== Rename commonly used method to a shorter version. To aid in source code readability. CanGetCharAtOffset -> CanGet PeekAheadChar -> Peek PeakChar -> Peek --- .../Portable/Scanner/Directives.vb | 12 ++--- .../VisualBasic/Portable/Scanner/Scanner.vb | 44 +++++++++---------- .../Portable/Scanner/ScannerBuffer.vb | 2 +- .../Portable/Scanner/ScannerXml.vb | 38 ++++++---------- .../Portable/Scanner/TokenFactories.vb | 2 +- .../Portable/Scanner/XmlDocComments.vb | 27 +++++------- .../Portable/Scanner/XmlTokenFactories.vb | 6 +-- 7 files changed, 55 insertions(+), 76 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb b/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb index 29ef1692c75c2..f8aaa56b2c5eb 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb @@ -31,8 +31,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim restorePoint = CreateRestorePoint() Me._isScanningDirective = True - ' since we do not have lookahead tokens, this just - ' resets current token to _lineBufferOffset + ' since we do not have lookahead tokens, this just + ' resets current token to _lineBufferOffset Me.GetNextTokenInState(ScannerState.VB) Dim currentNonterminal = Me.GetCurrentSyntaxNode() @@ -43,7 +43,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Me.MoveToNextSyntaxNodeInTrivia() ' adjust current token to just after the node - ' we need that in case we need to skip some disabled text + ' we need that in case we need to skip some disabled text '(yes we do tokenize disabled text for compatibility reasons) Me.GetNextTokenInState(ScannerState.VB) @@ -91,10 +91,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax disabledCode = SkipConditionalCompilationSection() End If - ' Here we add the directive and disabled text that follows it + ' Here we add the directive and disabled text that follows it ' (if there is any) to the trivia list - ' processing statement could add an error to it, + ' processing statement could add an error to it, ' so we may need to rebuild the trivia node If statement IsNot directiveTrivia Then directiveTrivia = statement @@ -507,7 +507,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '// '// Skip all text until the end of the current conditional section. This will also return the '// span of text that was skipped - '// + '// '//------------------------------------------------------------------------------------------------- Private Function SkipConditionalCompilationSection() As SyntaxList(Of VisualBasicSyntaxNode) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index b669b950320a3..d87d14d59f65a 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary @@ -37,8 +37,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private ReadOnly _sbPooled As PooledStringBuilder = PooledStringBuilder.GetInstance ''' - ''' DO NOT USE DIRECTLY. - ''' USE GetScratch() + ''' DO NOT USE DIRECTLY. + ''' USE GetScratch() ''' Private ReadOnly _sb As StringBuilder = _sbPooled.Builder Private ReadOnly _triviaListPool As New SyntaxListPool @@ -90,7 +90,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetScratch() As StringBuilder ' the normal pattern is that we clean scratch after use. - ' hitting this assert very likely indicates that you + ' hitting this assert very likely indicates that you ' did not release scratch content or worse trying to use ' scratch in two places at a time. Debug.Assert(_sb.Length = 0, "trying to use dirty buffer?") @@ -157,7 +157,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function GetNextToken(Optional allowLeadingMultilineTrivia As Boolean = False) As SyntaxToken - ' Use quick token scanning to see if we can scan a token quickly. + ' Use quick token scanning to see if we can scan a token quickly. Dim quickToken = QuickScanToken(allowLeadingMultilineTrivia) If quickToken.Succeeded Then @@ -240,6 +240,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' If we get too many characters that we cannot make sense of, absorb the rest of the input. token = MakeBadToken(leadingTrivia, RemainingLength(), ERRID.ERR_IllegalChar) End If + End If Return token @@ -324,7 +325,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Sub EatThroughLine() While CanGet() Dim c As Char = Peek() - If IsNewLine(c) Then EatThroughLineBreak(c) Return @@ -338,7 +338,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Gets a chunk of text as a DisabledCode node. ''' ''' The range of text. - ''' The DisabledCode node. + ''' The DisabledCode node. Friend Function GetDisabledTextAt(span As TextSpan) As SyntaxTrivia If span.Start >= 0 AndAlso span.End <= _bufferLen Then Return SyntaxFactory.DisabledTextTrivia(GetTextNotInterned(span.Start, span.Length)) @@ -489,9 +489,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function LengthOfLineBreak(StartCharacter As Char, Optional here As Integer = 0) As Integer Debug.Assert(CanGet(here)) Debug.Assert(IsNewLine(StartCharacter)) - Debug.Assert(StartCharacter = Peek(here)) - If StartCharacter = CARRIAGE_RETURN AndAlso NextIs(here + 1, LINE_FEED) Then Return 2 End If @@ -503,7 +501,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Accept a CR/LF pair or either in isolation as a newline. ''' Make it a statement separator - ''' + ''' Private Function ScanNewlineAsStatementTerminator(startCharacter As Char, precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken If _lineBufferOffset < _endOfTerminatorTrivia Then Dim width = LengthOfLineBreak(startCharacter) @@ -830,7 +828,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' 3. colon followed by new line -> colon terminator + new line terminator ' 4. new line followed by new line -> new line terminator + new line terminator - ' Case 3 is required to parse single line if's and numeric labels. + ' Case 3 is required to parse single line if's and numeric labels. ' Case 4 is required to limit explicit line continuations to single new line If CanGet() Then @@ -995,8 +993,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #End Region - ' at this point it is very likely that we are located at - ' the beginning of a token + ' at this point it is very likely that we are located at + ' the beginning of a token Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken If Not CanGet() Then @@ -1232,7 +1230,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return Nothing End Function - ' REVIEW: Is there a better way to reuse this logic? + ' REVIEW: Is there a better way to reuse this logic? Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken Select Case ch Case CARRIAGE_RETURN, LINE_FEED @@ -1465,7 +1463,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanRightAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean) As SyntaxToken - Debug.Assert(CanGet) ' > + Debug.Assert(CanGet) ' > Debug.Assert(Peek() = ">"c OrElse Peek() = FULLWIDTH_GREATER_THAN_SIGN) Dim length As Integer = 1 @@ -1492,7 +1490,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanLeftAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean, scanTrailingTrivia As ScanTriviaFunc) As SyntaxToken - Debug.Assert(CanGet) ' < + Debug.Assert(CanGet) ' < Debug.Assert(Peek() = "<"c OrElse Peek() = FULLWIDTH_LESS_THAN_SIGN) Dim length As Integer = 1 @@ -1568,10 +1566,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim c = spelling(0) If SyntaxFacts.IsIdentifierStartCharacter(c) Then - ' SPEC: ... Visual Basic identifiers conform to the Unicode Standard Annex 15 with one - ' SPEC: exception: identifiers may begin with an underscore (connector) character. - ' SPEC: If an identifier begins with an underscore, it must contain at least one other - ' SPEC: valid identifier character to disambiguate it from a line continuation. + ' SPEC: ... Visual Basic identifiers conform to the Unicode Standard Annex 15 with one + ' SPEC: exception: identifiers may begin with an underscore (connector) character. + ' SPEC: If an identifier begins with an underscore, it must contain at least one other + ' SPEC: valid identifier character to disambiguate it from a line continuation. If IsConnectorPunctuation(c) AndAlso spellingLength = 1 Then Return False End If @@ -2103,7 +2101,7 @@ FullWidthRepeat2: End If Else - ' // Copy the text of the literal to deal with fullwidth + ' // Copy the text of the literal to deal with fullwidth Dim scratch = GetScratch() For i = 0 To literalWithoutTypeChar - 1 Dim curCh = Peek(i) @@ -2251,7 +2249,7 @@ FullWidthRepeat2: HaveDateValue = True Here += 1 - ' Is the first value a year? + ' Is the first value a year? ' It is a year if it consists of exactly 4 digits. ' Condition below uses 5 because we already skipped the separator. If Here - FirstValueStart = 5 Then @@ -2598,7 +2596,7 @@ baddate: ' CC has trouble to prove this after the loop Debug.Assert(CanGet(length - 1)) - '// The literal does not have an explicit termination. + '// The literal does not have an explicit termination. ' DIFFERENT: here in IDE we used to report string token marked as unterminated Dim sp = GetTextNotInterned(length) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index 30e698a182c37..eb30658604ee0 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Imports System.Text diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb index 5e6e421df0beb..ae1cc8929a00f 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary Option Strict On @@ -148,7 +148,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeLessToken(leadingTrivia) Case "?"c - If NextIs(1,">"c) Then ' // Create token for the '?>' termination sequence Return XmlMakeEndProcessingInstructionToken(leadingTrivia) @@ -186,7 +185,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function '// - '// This is used to detect a VB statement on the next line + '// This is used to detect a VB statement on the next line '// '// NL WS* KW WS* ID | KW '// Example Dim x @@ -200,10 +199,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '// '// NL WS* < ID WS* ( '// Example ") Then - ' // If valid characters found then return them. If Here <> 0 Then Return XmlMakeTextLiteralToken(Nothing, Here, scratch) @@ -424,8 +421,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Case "%"c - 'TODO: error recovery. We cannot do this. - 'If there is all whitespace after ">", it will be scanned as insignificant, + 'TODO: error recovery. We cannot do this. + 'If there is all whitespace after ">", it will be scanned as insignificant, 'but in this case it is significant. 'Also as far as I can see Dev10 does not resync on "%>" text anyways. @@ -453,7 +450,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax GoTo ScanChars Case Else ScanChars: - ' // Check characters are valid + ' // Check characters are valid IsAllWhitespace = False Dim xmlCh = ScanXmlChar(Here) @@ -501,7 +498,6 @@ ScanChars: Case "-"c If NextIs(Here + 1,"-"c) Then - ' // --> terminates an Xml comment but otherwise -- is an illegal character sequence. ' // The scanner will always returns "--" as a separate comment data string and the ' // the semantics will error if '--' is ever found. @@ -526,9 +522,9 @@ ScanChars: ' // This works because the -> terminates only when the invalid -- ' // is returned. - 'If Here + 1 < m_InputStreamEnd AndAlso _ + 'If Here + 1 < m_InputStreamEnd AndAlso _ ' m_InputStream(Here) = "-"c AndAlso _ - ' m_InputStream(Here + 1) = ">"c Then + ' m_InputStream(Here + 1) = ">"c Then ' Here += 1 'Else @@ -670,7 +666,6 @@ ScanChars: Case "?"c If NextIs(Here + 1, ">"c) Then - '// If valid characters found then return them. If Here <> 0 Then result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here) @@ -752,7 +747,7 @@ CleanUp: Return XmlMakeLessToken(precedingTrivia) - ' TODO: review + ' TODO: review ' If Not m_State.m_ScannedElement OrElse c = "?"c OrElse c = "!"c Then ' ' // Remove tEOL from token ring if any exists @@ -984,7 +979,7 @@ CleanUp: End Function ''' - ''' 0 - not a surrogate, 2 - is valid surrogate + ''' 0 - not a surrogate, 2 - is valid surrogate ''' 1 is an error ''' Private Function ScanSurrogatePair(c1 As Char, Here As Integer) As XmlCharResult @@ -1003,7 +998,7 @@ CleanUp: Return Nothing End Function - ' contains result of Xml char scanning. + ' contains result of Xml char scanning. Friend Structure XmlCharResult Friend ReadOnly Length As Integer Friend ReadOnly Char1 As Char @@ -1162,7 +1157,6 @@ CreateNCNameToken: Case "a"c ' // & ' // ' - If CanGet(4) AndAlso NextAre(2,"mp") Then If Peek(4) = ";"c Then Return XmlMakeAmpLiteralToken(precedingTrivia) @@ -1185,9 +1179,7 @@ CreateNCNameToken: Case "l"c ' // < - If CanGet(3) AndAlso NextIs(2,"t"c) Then - If Peek(3) = ";"c Then Return XmlMakeLtLiteralToken(precedingTrivia) Else @@ -1199,9 +1191,7 @@ CreateNCNameToken: Case "g"c ' // > - If CanGet(3) AndAlso NextIs(2,"t"c) Then - If Peek(3) = ";"c Then Return XmlMakeGtLiteralToken(precedingTrivia) Else @@ -1213,9 +1203,7 @@ CreateNCNameToken: Case "q"c ' // " - If CanGet(5) AndAlso NextAre(2,"uot") Then - If Peek(5) = ";"c Then Return XmlMakeQuotLiteralToken(precedingTrivia) Else diff --git a/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb index 2f402543c7229..6231790bcd4c2 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index 175b5145f172f..215e742ca94d4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -49,7 +49,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function TryScanXmlDocComment(tList As SyntaxListBuilder) As Boolean Debug.Assert(IsAtNewLine) - ' leading whitespace until we see ''' should be regular whitespace If CanGet() AndAlso IsWhitespace(Peek()) Then Dim ws = ScanWhitespace() @@ -62,8 +61,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #End If Dim restorePoint = CreateRestorePoint() - ' since we do not have lookahead tokens, this just - ' resets current token to _lineBufferOffset + ' since we do not have lookahead tokens, this just + ' resets current token to _lineBufferOffset Me.GetNextTokenInState(ScannerState.Content) Dim currentNonterminal = Me.GetCurrentSyntaxNode() @@ -79,23 +78,23 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Me.IsScanningXmlDoc = True Me._isStartingFirstXmlDocLine = True - ' NOTE: Documentation comment syntax trivia must have at least one child xml node, because + ' NOTE: Documentation comment syntax trivia must have at least one child xml node, because ' all the ['''] trivia are created as leading trivia for appropriate tokens. ' This means that we have to create at least one XmlText having trailing ' EOL to represent an empty documentation comment: ['''] ' ' The problem with this approach is that in presence of some errors (like ' not closed XML tags) we create missing tokens needed to represent the nodes - ' *after* that last of the doc comment trivia, that means all the locations + ' *after* that last of the doc comment trivia, that means all the locations ' of created diagnostics will land on the first character of the next line ' after documentation comment ' - ' To workaround this we parse XML nodes in two phases: - ' - in the first phase we detect the last DocCommentLineBreak and create - ' end-of-xml token instead; this should force all diagnostics to be + ' To workaround this we parse XML nodes in two phases: + ' - in the first phase we detect the last DocCommentLineBreak and create + ' end-of-xml token instead; this should force all diagnostics to be ' reported on the next token location; - ' - in the second phase we continue parsing XML nodes but don't create - ' end-of-xml token which should just result in parsing one single node + ' - in the second phase we continue parsing XML nodes but don't create + ' end-of-xml token which should just result in parsing one single node ' of XmlText type containing EOL; ' Then we merge the results and create resulting DocumentationCommentTrivia @@ -106,7 +105,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' The second phase Me._endOfXmlInsteadOfLastDocCommentLineBreak = False If nodes.Count = 0 AndAlso parser.CurrentToken.Kind = SyntaxKind.EndOfXmlToken Then - ' This must be an empty documentation comment, we need to reset scanner so + ' This must be an empty documentation comment, we need to reset scanner so ' that the doc comment exterior trivia ([''']) lands on the final XmlNode ResetLineBufferOffset() @@ -255,7 +254,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If _endOfXmlInsteadOfLastDocCommentLineBreak Then Dim tempHere As Integer = Here If Not TrySkipXmlDocMarker(tempHere) Then - ' NOTE: we need to reset the buffer so that precedingTrivia + ' NOTE: we need to reset the buffer so that precedingTrivia ' lands on the next token ResetLineBufferOffset() Return SyntaxFactory.Token(Nothing, SyntaxKind.EndOfXmlToken, Nothing, String.Empty) @@ -313,7 +312,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Case "]"c If NextAre(Here + 1, "]>") Then - ' // If valid characters found then return them. If Here <> 0 Then Return XmlMakeTextLiteralToken(precedingTrivia, Here, scratch) @@ -325,7 +323,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax GoTo ScanChars Case Else ScanChars: - ' // Check characters are valid + ' // Check characters are valid Dim xmlCh = ScanXmlChar(Here) If xmlCh.Length = 0 Then @@ -397,7 +395,6 @@ ScanChars: Case "?"c If NextIs(Here + 1,">"c) Then - '// If valid characters found then return them. If Here <> 0 Then result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb index 5ed11a1fcb293..a384df6737af6 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary @@ -298,7 +298,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function XmlMakeBeginEndElementToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax Debug.Assert(NextAre("")) - AdvanceChar(2) Return MakePunctuationToken(SyntaxKind.SlashGreaterThanToken, "/>", precedingTrivia, Nothing) End Function @@ -354,7 +352,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function XmlMakeCloseBracketToken(state As ScannerState, precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As BadTokenSyntax Debug.Assert(Peek() = "]"c) - Return XmlMakeBadToken(SyntaxSubKind.CloseBracketToken, precedingTrivia, 1, If(state = ScannerState.DocType, ERRID.ERR_DTDNotSupported, ERRID.ERR_IllegalXmlNameChar)) End Function #End Region @@ -413,7 +410,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #Region "CData" Private Function XmlMakeBeginCDataToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax Debug.Assert(NextAre(" Date: Fri, 27 Mar 2015 20:42:10 +0000 Subject: [PATCH 2/9] Implemented `AreNext( chars : string ) : Boolean` method overload. Fixed the failing `Debug.Assert`s that used `AreNext(0, " ")` as the failure was being cause by CanGet looking to far ahead (off by one) --- src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index 215e742ca94d4..fd7c3deb6af9b 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -394,7 +394,7 @@ ScanChars: GoTo CleanUp Case "?"c - If NextIs(Here + 1,">"c) Then + If NextIs(Here + 1, ">"c) Then '// If valid characters found then return them. If Here <> 0 Then result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here) From 7d6acdbe9c663e14cdf45dd33f3a515fc1bd03c6 Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Tue, 31 Mar 2015 03:24:18 +0100 Subject: [PATCH 3/9] Added new extension methods, to for readability. --- .../Portable/Scanner/CharacterInfo.vb | 33 + .../Portable/Scanner/KeywordTable.vb | 4 +- .../VisualBasic/Portable/Scanner/Scanner.vb | 775 +++++------------- .../Portable/Scanner/ScannerBuffer.vb | 30 + .../Scanner/ScannerInterpolatedString.vb | 37 +- .../Portable/Scanner/ScannerXml.vb | 168 ++-- .../Portable/Scanner/XmlDocComments.vb | 49 +- .../Portable/Scanner/XmlTokenFactories.vb | 5 +- 8 files changed, 398 insertions(+), 703 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb index 099a979f87b7c..32fa7222eac95 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb @@ -488,4 +488,37 @@ Namespace Microsoft.CodeAnalysis.VisualBasic End Function End Class + Friend Module CharExts + + + Friend Function IsAnyOf(c As Char, c0 As Char, c1 As Char) As Boolean + Return (c = c0) OrElse (c = c1) + End Function + + + Friend Function IsAnyOf(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Return (c = c0) OrElse (c = c1) OrElse (c = c2) + End Function + + + Friend Function IsAnyOf(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Return (c = c0) OrElse (c = c1) OrElse (c = c2) OrElse (c = c3) + End Function + + + Friend Function IsNoneOf(c As Char, c0 As Char, c1 As Char) As Boolean + Return (c <> c0) AndAlso (c <> c1) + End Function + + + Friend Function IsNoneOf(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) + End Function + + + Friend Function IsNoneOf(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) AndAlso (c <> c3) + End Function + End Module + End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb index 47997330eb86d..7d896cca19342 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb @@ -304,9 +304,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax Private Shared Function EnsureHalfWidth(s As String) As String Dim result As Char() = Nothing - + Dim ch As Char For i As Integer = 0 To s.Length - 1 - Dim ch = s(i) + ch = s(i) If SyntaxFacts.IsFullWidth(ch) Then ch = SyntaxFacts.MakeHalfWidth(ch) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index d87d14d59f65a..26ab2e2557adb 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -225,8 +225,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanNextCharAsToken(leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim token As SyntaxToken - - If Not CanGet() Then + Dim c As Char + If Not TryPeek(c) Then token = MakeEofToken(leadingTrivia) Else _badTokenCount += 1 @@ -266,9 +266,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Dim condLineStart = _lineBufferOffset - - While (CanGet()) - Dim c As Char = Peek() + Dim c As Char + While TryPeek(c) Select Case (c) @@ -282,12 +281,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax EatWhitespace() Continue While - Case _ - "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, - "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, - "y"c, "z"c, "A"c, "B"c, "C"c, "D"c, "E"c, "F"c, "G"c, "H"c, "I"c, "J"c, - "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, "R"c, "S"c, "T"c, "U"c, "V"c, - "W"c, "X"c, "Y"c, "Z"c, "'"c, "_"c + Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, + "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, + "y"c, "z"c, "A"c, "B"c, "C"c, "D"c, "E"c, "F"c, "G"c, "H"c, "I"c, "J"c, + "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, "R"c, "S"c, "T"c, "U"c, "V"c, + "W"c, "X"c, "Y"c, "Z"c, "'"c, "_"c EatThroughLine() condLineStart = _lineBufferOffset @@ -326,12 +324,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax While CanGet() Dim c As Char = Peek() If IsNewLine(c) Then - EatThroughLineBreak(c) - Return - Else - AdvanceChar() - End If - End While + EatThroughLineBreak(c) + Return + Else + AdvanceChar() + End If + End While End Sub ''' @@ -487,7 +485,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function LengthOfLineBreak(StartCharacter As Char, Optional here As Integer = 0) As Integer - Debug.Assert(CanGet(here)) + Dim c As Char + Dim res = TryPeek(here, c) + Debug.Assert(res) Debug.Assert(IsNewLine(StartCharacter)) Debug.Assert(StartCharacter = Peek(here)) If StartCharacter = CARRIAGE_RETURN AndAlso NextIs(here + 1, LINE_FEED) Then @@ -531,27 +531,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanLineContinuation(tList As SyntaxListBuilder) As Boolean - If Not CanGet() Then - Return False - End If - - If Not IsAfterWhitespace() Then - Return False - End If - - Dim ch As Char = Peek() - If Not IsUnderscore(ch) Then + Dim ch As Char + If Not TryPeek(ch) OrElse Not IsAfterWhitespace() OrElse Not IsUnderscore(ch) Then Return False End If Dim Here = 1 - While CanGet(Here) - ch = Peek(Here) - If IsWhitespace(ch) Then - Here += 1 - Else - Exit While - End If + While TryPeek(Here, ch) AndAlso IsWhitespace(ch) + Here += 1 End While ' Line continuation is valid at the end of the @@ -576,9 +563,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' do not include the new line character since that would confuse code handling ' implicit line continuations. (See Scanner::EatLineContinuation.) Otherwise, ' include the new line and any additional spaces as trivia. - If startComment = 0 AndAlso - CanGet(Here) AndAlso - Not IsNewLine(Peek(Here)) Then + + If startComment = 0 AndAlso TryPeek(Here, ch) AndAlso Not IsNewLine(ch) Then tList.Add(MakeEndOfLineTrivia(GetText(newLine))) If spaces > 0 Then @@ -599,15 +585,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Consumes all trivia until a nontrivia char is found ''' Friend Function ScanMultilineTrivia() As SyntaxList(Of VisualBasicSyntaxNode) - If Not CanGet() Then + Dim ch As Char + If Not TryPeek(ch) Then Return Nothing End If - Dim ch = Peek() - ' optimization for a common case ' the ASCII range between ': and ~ , with exception of except "'", "_" and R cannot start trivia - If ch > ":"c AndAlso ch <= "~"c AndAlso ch <> "'"c AndAlso ch <> "_"c AndAlso ch <> "R"c AndAlso ch <> "r"c Then + If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsNoneOf("'"c, "_"c, "R"c, "r"c) Then Return Nothing End If @@ -624,7 +609,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Scans a single piece of trivia ''' Private Function TryScanSinglePieceOfMultilineTrivia(tList As SyntaxListBuilder) As Boolean - If CanGet() Then + Dim ch As Char + If TryPeek(ch) Then Dim atNewLine = IsAtNewLine() @@ -639,7 +625,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If End If - Dim ch = Peek() If IsWhitespace(ch) Then ' eat until linebreak or nonwhitespace Dim wslen = GetWhitespaceLength(1) @@ -684,11 +669,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' check for # Private Function StartsDirective(Here As Integer) As Boolean - If CanGet(Here) Then - Dim ch = Peek(Here) - Return IsHash(ch) - End If - Return False + Dim ch As Char + Return TryPeek(Here, ch) AndAlso IsHash(ch) End Function Private Function IsAtNewLine() As Boolean @@ -728,8 +710,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Sub Private Sub ScanSingleLineTriviaInXmlDoc(tList As SyntaxListBuilder) - If CanGet() Then - Dim c As Char = Peek() + Dim c As Char + If TryPeek(c) Then Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -761,7 +743,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Sub ScanWhitespaceAndLineContinuations(tList As SyntaxListBuilder) - If CanGet() AndAlso IsWhitespace(Peek()) Then + Dim c As Char + If TryPeek(c) AndAlso IsWhitespace(c) Then tList.Add(ScanWhitespace(1)) ' collect { lineCont, ws } While ScanLineContinuation(tList) @@ -830,10 +813,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' Case 3 is required to parse single line if's and numeric labels. ' Case 4 is required to limit explicit line continuations to single new line + Dim ch As Char + If TryPeek(ch) Then - If CanGet() Then - - Dim ch As Char = Peek() Dim startOfTerminatorTrivia = _lineBufferOffset If IsNewLine(ch) Then @@ -845,11 +827,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' collect { ws, colon } Do Dim len = GetWhitespaceLength(0) - If Not CanGet(len) Then + If Not TryPeek(len, ch) Then Exit Do End If - ch = Peek(len) If Not IsColonAndNotColonEquals(ch, offset:=len) Then Exit Do End If @@ -885,7 +866,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetWhitespaceLength(len As Integer) As Integer ' eat until linebreak or nonwhitespace - While CanGet(len) AndAlso IsWhitespace(Peek(len)) + Dim c As Char + While TryPeek(len, c) AndAlso IsWhitespace(c) len += 1 End While Return len @@ -893,7 +875,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetXmlWhitespaceLength(len As Integer) As Integer ' eat until linebreak or nonwhitespace - While CanGet(len) AndAlso IsXmlWhitespace(Peek(len)) + Dim c As Char + While TryPeek(len, c) AndAlso IsXmlWhitespace(c) len += 1 End While Return len @@ -916,32 +899,33 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Sub EatWhitespace() - Debug.Assert(CanGet) - Debug.Assert(IsWhitespace(Peek())) + Dim c As Char + Dim ok = TryPeek(c) + Debug.Assert(ok) + Debug.Assert(IsWhitespace(c)) AdvanceChar() ' eat until linebreak or nonwhitespace - While CanGet() AndAlso IsWhitespace(Peek) + While TryPeek(c) AndAlso IsWhitespace(c) AdvanceChar() End While End Sub Private Function PeekStartComment(i As Integer) As Integer - - If CanGet(i) Then - Dim ch = Peek(i) - + Dim ch As Char + If TryPeek(i, ch) Then + Dim c2, c3 As Char If IsSingleQuote(ch) Then Return 1 ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso - CanGet(i + 2) AndAlso MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso - MatchOneOrAnotherOrFullwidth(Peek(i + 2), "M"c, "m"c) Then + TryPeek(i + 2, c2) AndAlso MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso + MatchOneOrAnotherOrFullwidth(c2, "M"c, "m"c) Then - If Not CanGet(i + 3) OrElse IsNewLine(Peek(i + 3)) Then + If Not TryPeek(i + 3, c3) OrElse IsNewLine(c3) Then ' have only 'REM' Return 3 - ElseIf Not IsIdentifierPartCharacter(Peek(i + 3)) Then + ElseIf Not IsIdentifierPartCharacter(c3) Then ' have 'REM ' Return 4 End If @@ -959,9 +943,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim looksLikeDocComment As Boolean = StartsXmlDoc(0) ' eat all chars until EoL - While CanGet(length) AndAlso - Not IsNewLine(Peek(length)) - + Dim c As Char + While TryPeek(length, c) AndAlso Not IsNewLine(c) length += 1 End While @@ -995,448 +978,193 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' at this point it is very likely that we are located at ' the beginning of a token - Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - - If Not CanGet() Then - Return MakeEofToken(precedingTrivia) - End If + Private Function ScanTokenCommon(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char, fullWidth As Boolean) As SyntaxToken + Dim lengthWithMaybeEquals As Integer = 1 + Dim c As Char - Dim ch As Char = Peek() Select Case ch - Case CARRIAGE_RETURN, LINE_FEED, NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR + Case CARRIAGE_RETURN, LINE_FEED + Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) + Case NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR + If fullWidth Then Exit Select Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) - Case " "c, CHARACTER_TABULATION, "'"c Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch))) Return Nothing ' trivia cannot start a token - Case "@"c - Return MakeAtToken(precedingTrivia, False) - + Return MakeAtToken(precedingTrivia, fullWidth) Case "("c - Return MakeOpenParenToken(precedingTrivia, False) - + Return MakeOpenParenToken(precedingTrivia, fullWidth) Case ")"c - Return MakeCloseParenToken(precedingTrivia, False) - + Return MakeCloseParenToken(precedingTrivia, fullWidth) Case "{"c - Return MakeOpenBraceToken(precedingTrivia, False) - + Return MakeOpenBraceToken(precedingTrivia, fullWidth) Case "}"c - Return MakeCloseBraceToken(precedingTrivia, False) - + Return MakeCloseBraceToken(precedingTrivia, fullWidth) Case ","c - Return MakeCommaToken(precedingTrivia, False) - + Return MakeCommaToken(precedingTrivia, fullWidth) Case "#"c Dim dl = ScanDateLiteral(precedingTrivia) - If dl IsNot Nothing Then - Return dl - Else - Return MakeHashToken(precedingTrivia, False) - End If - + Return If(dl, MakeHashToken(precedingTrivia, fullWidth)) Case "&"c - If CanGet(1) AndAlso BeginsBaseLiteral(Peek(1)) Then + If TryPeek(1, c) AndAlso BeginsBaseLiteral(c) Then Return ScanNumericLiteral(precedingTrivia) End If - - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeAmpersandEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeAmpersandToken(precedingTrivia, False) + Return MakeAmpersandToken(precedingTrivia, fullWidth) End If - Case "="c - Return MakeEqualsToken(precedingTrivia, False) - + Return MakeEqualsToken(precedingTrivia, fullWidth) Case "<"c - Return ScanLeftAngleBracket(precedingTrivia, False, _scanSingleLineTriviaFunc) - + Return ScanLeftAngleBracket(precedingTrivia, fullWidth, _scanSingleLineTriviaFunc) Case ">"c - Return ScanRightAngleBracket(precedingTrivia, False) - + Return ScanRightAngleBracket(precedingTrivia, fullWidth) Case ":"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeColonEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return ScanColonAsStatementTerminator(precedingTrivia, False) + Return ScanColonAsStatementTerminator(precedingTrivia, fullWidth) End If - Case "+"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakePlusEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakePlusToken(precedingTrivia, False) + Return MakePlusToken(precedingTrivia, fullWidth) End If - Case "-"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeMinusEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeMinusToken(precedingTrivia, False) + Return MakeMinusToken(precedingTrivia, fullWidth) End If - Case "*"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeAsteriskEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeAsteriskToken(precedingTrivia, False) + Return MakeAsteriskToken(precedingTrivia, fullWidth) End If - Case "/"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeSlashToken(precedingTrivia, False) + Return MakeSlashToken(precedingTrivia, fullWidth) End If - Case "\"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeBackSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeBackslashToken(precedingTrivia, False) + Return MakeBackslashToken(precedingTrivia, fullWidth) End If - Case "^"c - Dim lengthWithMaybeEquals = 1 If TrySkipFollowingEquals(lengthWithMaybeEquals) Then Return MakeCaretEqualsToken(precedingTrivia, lengthWithMaybeEquals) Else - Return MakeCaretToken(precedingTrivia, False) + Return MakeCaretToken(precedingTrivia, fullWidth) End If - Case "!"c - Return MakeExclamationToken(precedingTrivia, False) - + Return MakeExclamationToken(precedingTrivia, fullWidth) Case "."c If CanGet(1) AndAlso IsDecimalDigit(Peek(1)) Then Return ScanNumericLiteral(precedingTrivia) Else - Return MakeDotToken(precedingTrivia, False) + Return MakeDotToken(precedingTrivia, fullWidth) End If - - Case "0"c, - "1"c, - "2"c, - "3"c, - "4"c, - "5"c, - "6"c, - "7"c, - "8"c, - "9"c + Case "0"c, "1"c, "2"c, "3"c, "4"c, + "5"c, "6"c, "7"c, "8"c, "9"c Return ScanNumericLiteral(precedingTrivia) - Case """"c Return ScanStringLiteral(precedingTrivia) - Case "A"c If NextAre(1, "s ") Then - ' TODO: do we allow widechars in keywords? Dim spelling = "As" - AdvanceChar(2) + If fullWidth Then spelling = GetText(2) Else AdvanceChar(2) Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If - Case "E"c If NextAre(1, "nd ") Then - ' TODO: do we allow widechars in keywords? Dim spelling = "End" - AdvanceChar(3) + If fullWidth Then spelling = GetText(3) Else AdvanceChar(3) Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If - Case "I"c If NextAre(1, "f ") Then - ' TODO: do we allow widechars in keywords? Dim spelling = "If" - AdvanceChar(2) + If fullWidth Then spelling = GetText(2) Else AdvanceChar(2) Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If - Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, "y"c, "z"c Return ScanIdentifierOrKeyword(precedingTrivia) - Case "B"c, "C"c, "D"c, "F"c, "G"c, "H"c, "J"c, "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, - "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c + "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c Return ScanIdentifierOrKeyword(precedingTrivia) - Case "_"c - If CanGet(1) AndAlso IsIdentifierPartCharacter(Peek(1)) Then + If TryPeek(1, c) AndAlso IsIdentifierPartCharacter(c) Then Return ScanIdentifierOrKeyword(precedingTrivia) End If - Dim err As ERRID = ERRID.ERR_ExpectedIdentifier Dim len = GetWhitespaceLength(1) If Not CanGet(len) OrElse IsNewLine(Peek(len)) OrElse PeekStartComment(len) > 0 Then err = ERRID.ERR_LineContWithCommentOrNoPrecSpace End If - ' not a line continuation and cannot start identifier. Return MakeBadToken(precedingTrivia, 1, err) - Case "["c Return ScanBracketedIdentifier(precedingTrivia) - Case "?"c - Return MakeQuestionToken(precedingTrivia, False) - + Return MakeQuestionToken(precedingTrivia, fullWidth) Case "%"c - If CanGet(1) AndAlso - Peek(1) = ">"c Then + If NextIs(1, ">"c) Then Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc) End If - Case "$"c, FULLWIDTH_DOLLAR_SIGN - If CanGet(1) AndAlso IsDoubleQuote(Peek(1)) Then + If fullWidth Then Exit Select + If TryPeek(1, c) AndAlso IsDoubleQuote(c) Then Return MakePunctuationToken(precedingTrivia, 2, SyntaxKind.DollarSignDoubleQuoteToken) End If - End Select - If IsIdentifierStartCharacter(ch) Then Return ScanIdentifierOrKeyword(precedingTrivia) End If - Debug.Assert(Not IsNewLine(ch)) - - If IsDoubleQuote(ch) Then - Return ScanStringLiteral(precedingTrivia) - End If - - If IsFullWidth(ch) Then - ch = MakeHalfWidth(ch) - Return ScanTokenFullWidth(precedingTrivia, ch) + If fullWidth Then + Debug.Assert(Not IsDoubleQuote(ch)) + Else + If IsDoubleQuote(ch) Then + Return ScanStringLiteral(precedingTrivia) + End If + If IsFullWidth(ch) Then + ch = MakeHalfWidth(ch) + Return ScanTokenFullWidth(precedingTrivia, ch) + End If End If - Return Nothing End Function - ' REVIEW: Is there a better way to reuse this logic? - Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken - Select Case ch - Case CARRIAGE_RETURN, LINE_FEED - Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) - - Case " "c, CHARACTER_TABULATION, "'"c - Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch))) - Return Nothing ' trivia cannot start a token - - Case "@"c - Return MakeAtToken(precedingTrivia, True) - - Case "("c - Return MakeOpenParenToken(precedingTrivia, True) - - Case ")"c - Return MakeCloseParenToken(precedingTrivia, True) - - Case "{"c - Return MakeOpenBraceToken(precedingTrivia, True) - - Case "}"c - Return MakeCloseBraceToken(precedingTrivia, True) - - Case ","c - Return MakeCommaToken(precedingTrivia, True) - - Case "#"c - Dim dl = ScanDateLiteral(precedingTrivia) - If dl IsNot Nothing Then - Return dl - Else - Return MakeHashToken(precedingTrivia, True) - End If - - Case "&"c - If CanGet(1) AndAlso BeginsBaseLiteral(Peek(1)) Then - Return ScanNumericLiteral(precedingTrivia) - End If - - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeAmpersandEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeAmpersandToken(precedingTrivia, True) - End If - - Case "="c - Return MakeEqualsToken(precedingTrivia, True) - - Case "<"c - Return ScanLeftAngleBracket(precedingTrivia, True, _scanSingleLineTriviaFunc) - - Case ">"c - Return ScanRightAngleBracket(precedingTrivia, True) - - Case ":"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeColonEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return ScanColonAsStatementTerminator(precedingTrivia, True) - End If - - Case "+"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakePlusEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakePlusToken(precedingTrivia, True) - End If - - Case "-"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeMinusEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeMinusToken(precedingTrivia, True) - End If - - Case "*"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeAsteriskEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeAsteriskToken(precedingTrivia, True) - End If - - Case "/"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeSlashToken(precedingTrivia, True) - End If - - Case "\"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeBackSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeBackslashToken(precedingTrivia, True) - End If - - Case "^"c - Dim lengthWithMaybeEquals = 1 - If TrySkipFollowingEquals(lengthWithMaybeEquals) Then - Return MakeCaretEqualsToken(precedingTrivia, lengthWithMaybeEquals) - Else - Return MakeCaretToken(precedingTrivia, True) - End If - - Case "!"c - Return MakeExclamationToken(precedingTrivia, True) - - Case "."c - If CanGet(1) AndAlso IsDecimalDigit(Peek(1)) Then - Return ScanNumericLiteral(precedingTrivia) - Else - Return MakeDotToken(precedingTrivia, True) - End If - - Case "0"c, - "1"c, - "2"c, - "3"c, - "4"c, - "5"c, - "6"c, - "7"c, - "8"c, - "9"c - Return ScanNumericLiteral(precedingTrivia) - - Case """"c - Return ScanStringLiteral(precedingTrivia) - - Case "A"c - If NextAre(1, "s ") Then - Dim spelling = GetText(2) - Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia) - Else - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Case "E"c - If NextAre(1, "nd ") Then - Dim spelling = GetText(3) - Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia) - Else - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Case "I"c - If NextAre(1, "f ") Then - ' TODO: do we allow widechars in keywords? - Dim spelling = GetText(2) - Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia) - Else - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, "m"c, - "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, "y"c, "z"c - Return ScanIdentifierOrKeyword(precedingTrivia) - - Case "B"c, "C"c, "D"c, "F"c, "G"c, "H"c, "J"c, "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, - "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c - Return ScanIdentifierOrKeyword(precedingTrivia) - - Case "_"c - If CanGet(1) AndAlso IsIdentifierPartCharacter(Peek(1)) Then - Return ScanIdentifierOrKeyword(precedingTrivia) - End If - - Dim err As ERRID = ERRID.ERR_ExpectedIdentifier - Dim len = GetWhitespaceLength(1) - If Not CanGet(len) OrElse IsNewLine(Peek(len)) OrElse PeekStartComment(len) > 0 Then - err = ERRID.ERR_LineContWithCommentOrNoPrecSpace - End If - - ' not a line continuation and cannot start identifier. - Return MakeBadToken(precedingTrivia, 1, err) - - Case "["c - Return ScanBracketedIdentifier(precedingTrivia) - - Case "?"c - Return MakeQuestionToken(precedingTrivia, True) - - Case "%"c - If CanGet(1) AndAlso - Peek(1) = ">"c Then - Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc) - End If - - End Select - If IsIdentifierStartCharacter(ch) Then - Return ScanIdentifierOrKeyword(precedingTrivia) + ' at this point it is very likely that we are located at + ' the beginning of a token + Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken + Dim ch As Char + If Not TryPeek(ch) Then + Return MakeEofToken(precedingTrivia) End If + Return ScanTokenCommon(precedingTrivia, ch, False) + End Function - Debug.Assert(Not IsNewLine(ch)) - Debug.Assert(Not IsDoubleQuote(ch)) - - Return Nothing + Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken + Return ScanTokenCommon(precedingTrivia, ch, True) End Function ' // Allow whitespace between the characters of a two-character token. @@ -1447,11 +1175,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim Here = Index Dim eq As Char - While CanGet(Here) - eq = Peek(Here) + While TryPeek(Here, eq) Here += 1 If Not IsWhitespace(eq) Then - If eq = "="c OrElse eq = FULLWIDTH_EQUALS_SIGN Then + If eq.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then Index = Here Return True Else @@ -1463,21 +1190,20 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanRightAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean) As SyntaxToken - Debug.Assert(CanGet) ' > - Debug.Assert(Peek() = ">"c OrElse Peek() = FULLWIDTH_GREATER_THAN_SIGN) - + Dim c As Char + Dim ok = TryPeek(c) + Debug.Assert(ok) ' > + Debug.Assert(c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN)) Dim length As Integer = 1 ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - If CanGet(length) Then - Dim c As Char = Peek(length) - - If c = "="c OrElse c = FULLWIDTH_EQUALS_SIGN Then + If TryPeek(length, c) Then + If c.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 Return MakeGreaterThanEqualsToken(precedingTrivia, length) - ElseIf c = ">"c OrElse c = FULLWIDTH_GREATER_THAN_SIGN Then + ElseIf c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then length += 1 If TrySkipFollowingEquals(length) Then Return MakeGreaterThanGreaterThanEqualsToken(precedingTrivia, length) @@ -1490,26 +1216,27 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanLeftAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean, scanTrailingTrivia As ScanTriviaFunc) As SyntaxToken - Debug.Assert(CanGet) ' < - Debug.Assert(Peek() = "<"c OrElse Peek() = FULLWIDTH_LESS_THAN_SIGN) - + Dim c As Char + Dim ok = TryPeek(c) + Debug.Assert(ok) ' < + Debug.Assert(c.IsAnyOf("<"c, FULLWIDTH_LESS_THAN_SIGN)) Dim length As Integer = 1 ' Check for XML tokens - If Not charIsFullWidth AndAlso CanGet(length) Then - Dim c As Char = Peek(length) + If Not charIsFullWidth AndAlso TryPeek(length, c) Then + Select Case c Case "!"c - If CanGet(length + 2) Then + Dim c2 As Char + If TryPeek(length + 2, c2) Then Select Case (Peek(length + 1)) Case "-"c - If CanGet(length + 3) AndAlso Peek(length + 2) = "-"c Then + If CanGet(length + 3) AndAlso c2 = "-"c Then Return XmlMakeBeginCommentToken(precedingTrivia, scanTrailingTrivia) End If - Case "["c + Case "["c If NextAre(length + 2, "CDATA[") Then - Return XmlMakeBeginCDataToken(precedingTrivia, scanTrailingTrivia) End If End Select @@ -1525,23 +1252,19 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - If CanGet(length) Then - Dim c As Char = Peek(length) - If c = "="c OrElse c = FULLWIDTH_EQUALS_SIGN Then + If TryPeek(length, c) Then + If c.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 Return MakeLessThanEqualsToken(precedingTrivia, length) - ElseIf c = ">"c OrElse c = FULLWIDTH_GREATER_THAN_SIGN Then + ElseIf c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then length += 1 Return MakeLessThanGreaterThanToken(precedingTrivia, length) - ElseIf c = "<"c OrElse c = FULLWIDTH_LESS_THAN_SIGN Then + ElseIf c.IsAnyOf("<"c, FULLWIDTH_LESS_THAN_SIGN) Then length += 1 - - If CanGet(length) Then - c = Peek(length) - + If TryPeek(length, c) Then 'if the second "<" is a part of "<%" - like in "<<%" , we do not want to use it. - If c <> "%"c AndAlso c <> FULLWIDTH_PERCENT_SIGN Then + If c.IsNoneOf("%"c, FULLWIDTH_PERCENT_SIGN) Then If TrySkipFollowingEquals(length) Then Return MakeLessThanLessThanEqualsToken(precedingTrivia, length) Else @@ -1585,13 +1308,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanIdentifierOrKeyword(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Debug.Assert(CanGet) - Debug.Assert(IsIdentifierStartCharacter(Peek)) + Dim ch As Char + Dim ok = TryPeek(ch) + Debug.Assert(ok) + Debug.Assert(IsIdentifierStartCharacter(ch)) Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller - - Dim ch = Peek() - If CanGet(1) Then - Dim ch1 = Peek(1) + Dim ch1 As Char + If TryPeek(1, ch1) Then If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(ch1) Then Return MakeBadToken(precedingTrivia, 1, ERRID.ERR_ExpectedIdentifier) End If @@ -1602,8 +1325,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // The C++ compiler refuses to inline IsIdentifierCharacter, so the ' // < 128 test is inline here. (This loop gets a *lot* of traffic.) ' TODO: make sure we get good perf here - While CanGet(len) - ch = Peek(len) + While TryPeek(len, ch) Dim code = Convert.ToUInt16(ch) If code < 128 AndAlso IsNarrowIdentifierCharacter(code) OrElse @@ -1617,16 +1339,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax 'Check for a type character Dim TypeCharacter As TypeCharacter = TypeCharacter.None - If CanGet(len) Then - ch = Peek(len) + If TryPeek(len, ch) Then FullWidthRepeat: Select Case ch Case "!"c + Dim NextChar As Char ' // If the ! is followed by an identifier it is a dictionary lookup operator, not a type character. - If CanGet(len + 1) Then - Dim NextChar As Char = Peek(len + 1) - + If TryPeek(len + 1, NextChar) Then If IsIdentifierStartCharacter(NextChar) OrElse MatchOneOrAnotherOrFullwidth(NextChar, "["c, "]"c) Then Exit Select @@ -1667,9 +1387,7 @@ FullWidthRepeat: Dim contextualKind As SyntaxKind = SyntaxKind.IdentifierToken Dim spelling = GetText(len) - Dim BaseSpelling = If(TypeCharacter = TypeCharacter.None, - spelling, - Intern(spelling, 0, len - 1)) + Dim BaseSpelling = If(TypeCharacter = TypeCharacter.None, spelling, Intern(spelling, 0, len - 1)) ' this can be keyword only if it has no type character, or if it is Mid$ If TypeCharacter = TypeCharacter.None Then @@ -1703,34 +1421,35 @@ FullWidthRepeat: End Function Private Function ScanBracketedIdentifier(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Debug.Assert(CanGet) ' [ - Debug.Assert(Peek() = "["c OrElse Peek() = FULLWIDTH_LEFT_SQUARE_BRACKET) + Dim ch As Char + Dim ok = TryPeek(ch) + Debug.Assert(ok) ' [ + Debug.Assert(ch.IsAnyOf("["c, FULLWIDTH_LEFT_SQUARE_BRACKET)) Dim IdStart As Integer = 1 Dim Here As Integer = IdStart Dim InvalidIdentifier As Boolean = False - If Not CanGet(Here) Then + If Not TryPeek(Here, ch) Then Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_MissingEndBrack) End If - Dim ch = Peek(Here) + Dim c1 As Char ' check if we can start an ident. If Not IsIdentifierStartCharacter(ch) OrElse (IsConnectorPunctuation(ch) AndAlso - Not (CanGet(Here + 1) AndAlso - IsIdentifierPartCharacter(Peek(Here + 1)))) Then + Not (TryPeek(Here + 1, c1) AndAlso IsIdentifierPartCharacter(c1))) Then InvalidIdentifier = True End If ' check ident until ] - While CanGet(Here) - Dim [Next] As Char = Peek(Here) + Dim [Next] As Char + While TryPeek(Here, [Next]) - If [Next] = "]"c OrElse [Next] = FULLWIDTH_RIGHT_SQUARE_BRACKET Then + If [Next].IsAnyOf("]"c, FULLWIDTH_RIGHT_SQUARE_BRACKET) Then Dim IdStringLength As Integer = Here - IdStart If IdStringLength > 0 AndAlso Not InvalidIdentifier Then @@ -1776,7 +1495,9 @@ FullWidthRepeat: End Enum Private Function ScanNumericLiteral(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Debug.Assert(CanGet) + Dim ch As Char + Dim ok = TryPeek(ch) + Debug.Assert(ok) Dim Here As Integer = 0 Dim IntegerLiteralStart As Integer @@ -1790,10 +1511,9 @@ FullWidthRepeat: ' // First read a leading base specifier, if present, followed by a sequence of zero ' // or more digits. - Dim ch = Peek() - If ch = "&"c OrElse ch = FULLWIDTH_AMPERSAND Then + If ch.IsAnyOf("&"c, FULLWIDTH_AMPERSAND) Then Here += 1 - ch = If(CanGet(Here), Peek(Here), ChrW(0)) + If Not TryPeek(Here, ch) Then ch = ChrW(0) FullWidthRepeat: Select Case ch @@ -1802,11 +1522,7 @@ FullWidthRepeat: IntegerLiteralStart = Here Base = LiteralBase.Hexadecimal - While CanGet(Here) - ch = Peek(Here) - If Not IsHexDigit(ch) Then - Exit While - End If + While TryPeek(Here, ch) AndAlso IsHexDigit(ch) Here += 1 End While @@ -1815,11 +1531,7 @@ FullWidthRepeat: IntegerLiteralStart = Here Base = LiteralBase.Octal - While CanGet(Here) - ch = Peek(Here) - If Not IsOctalDigit(ch) Then - Exit While - End If + While TryPeek(Here, ch) AndAlso IsOctalDigit(ch) Here += 1 End While @@ -1834,11 +1546,7 @@ FullWidthRepeat: Else ' no base specifier - just go through decimal digits. IntegerLiteralStart = Here - While CanGet(Here) - ch = Peek(Here) - If Not IsDecimalDigit(ch) Then - Exit While - End If + While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While End If @@ -1848,21 +1556,16 @@ FullWidthRepeat: ' // Unless there was an explicit base specifier (which indicates an integer literal), ' // read the rest of a float literal. - If Base = LiteralBase.Decimal AndAlso CanGet(Here) Then + If Base = LiteralBase.Decimal AndAlso TryPeek(Here, ch) Then ' // First read a '.' followed by a sequence of one or more digits. - ch = Peek(Here) - If (ch = "."c Or ch = FULLWIDTH_FULL_STOP) AndAlso - CanGet(Here + 1) AndAlso - IsDecimalDigit(Peek(Here + 1)) Then + Dim cx As Char + If ch.IsAnyOf("."c, FULLWIDTH_FULL_STOP) AndAlso + (TryPeek(Here + 1, cx) AndAlso IsDecimalDigit(cx)) Then Here += 2 ' skip dot and first digit ' all following decimal digits belong to the literal (fractional part) - While CanGet(Here) - ch = Peek(Here) - If Not IsDecimalDigit(ch) Then - Exit While - End If + While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While literalKind = NumericLiteralKind.Float @@ -1870,24 +1573,16 @@ FullWidthRepeat: ' // Read an exponent symbol followed by an optional sign and a sequence of ' // one or more digits. - If CanGet(Here) AndAlso BeginsExponent(Peek(Here)) Then + If TryPeek(Here, cx) AndAlso BeginsExponent(cx) Then Here += 1 - If CanGet(Here) Then - ch = Peek(Here) - - If MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then - Here += 1 - End If + If TryPeek(Here, ch) AndAlso MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then + Here += 1 End If - If CanGet(Here) AndAlso IsDecimalDigit(Peek(Here)) Then + If TryPeek(Here, cx) AndAlso IsDecimalDigit(cx) Then Here += 1 - While CanGet(Here) - ch = Peek(Here) - If Not IsDecimalDigit(ch) Then - Exit While - End If + While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While Else @@ -1906,8 +1601,7 @@ FullWidthRepeat: Dim TypeCharacter As TypeCharacter = TypeCharacter.None - If CanGet(Here) Then - ch = Peek(Here) + If TryPeek(Here, ch) Then FullWidthRepeat2: Select Case ch @@ -1983,9 +1677,7 @@ FullWidthRepeat2: literalKind = NumericLiteralKind.Decimal ' check if this was not attempt to use obsolete exponent - If CanGet(Here + 1) Then - ch = Peek(Here + 1) - + If TryPeek(Here + 1, ch) Then If IsDecimalDigit(ch) OrElse MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_ObsoleteExponent) End If @@ -1995,9 +1687,8 @@ FullWidthRepeat2: End If Case "U"c, "u"c - If literalKind <> NumericLiteralKind.Float AndAlso CanGet(Here + 1) Then - Dim NextChar As Char = Peek(Here + 1) - + Dim NextChar As Char + If literalKind <> NumericLiteralKind.Float AndAlso TryPeek(Here + 1, NextChar) Then 'unsigned suffixes - US, UL, UI If MatchOneOrAnotherOrFullwidth(NextChar, "S"c, "s"c) Then TypeCharacter = TypeCharacter.UShortLiteral @@ -2168,31 +1859,17 @@ FullWidthRepeat2: Return Decimal.TryParse(text, NumberStyles.AllowDecimalPoint Or NumberStyles.AllowExponent, CultureInfo.InvariantCulture, value) End Function - Private Function ScanIntLiteral( - ByRef ReturnValue As Integer, - ByRef Here As Integer - ) As Boolean + Private Function ScanIntLiteral(ByRef ReturnValue As Integer, ByRef Here As Integer) As Boolean Debug.Assert(Here >= 0) - - If Not CanGet(Here) Then - Return False - End If - - Dim ch = Peek(Here) - If Not IsDecimalDigit(ch) Then + Dim ch As Char + If Not TryPeek(Here, ch) OrElse Not IsDecimalDigit(ch) Then Return False End If Dim IntegralValue As Integer = IntegralLiteralCharacterValue(ch) Here += 1 - While CanGet(Here) - ch = Peek(Here) - - If Not IsDecimalDigit(ch) Then - Exit While - End If - + While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) Dim nextDigit = IntegralLiteralCharacterValue(ch) If IntegralValue < 214748364 OrElse (IntegralValue = 214748364 AndAlso nextDigit < 8) Then @@ -2241,8 +1918,9 @@ FullWidthRepeat2: End If ' // If we see a /, then it's a date + Dim c As Char - If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then + If TryPeek(Here, c) AndAlso IsDateSeparatorCharacter(c) Then Dim FirstDateSeparator As Integer = Here ' // We've got a date @@ -2262,10 +1940,11 @@ FullWidthRepeat2: End If ' Do we have a day value? - If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then + Dim c1 As Char + If TryPeek(Here, c1) AndAlso IsDateSeparatorCharacter(c1) Then ' // Check to see they used a consistent separator - If Peek(Here) <> Peek(FirstDateSeparator) Then + If c1 <> Peek(FirstDateSeparator) Then GoTo baddate End If @@ -2287,11 +1966,11 @@ FullWidthRepeat2: End If ' // Do we have a year value? - - If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then + Dim c1 As Char + If TryPeek(Here, c1) AndAlso IsDateSeparatorCharacter(c1) Then ' // Check to see they used a consistent separator - If Peek(Here) <> Peek(FirstDateSeparator) Then + If c1 <> Peek(FirstDateSeparator) Then GoTo baddate End If @@ -2330,8 +2009,8 @@ FullWidthRepeat2: If HaveTimeValue Then ' // Do we see a :? - - If CanGet(Here) AndAlso IsColon(Peek(Here)) Then + Dim c1 As Char + If TryPeek(Here, c1) AndAlso IsColon(c1) Then Here += 1 ' // Now let's get the minute value @@ -2343,8 +2022,8 @@ FullWidthRepeat2: HaveMinuteValue = True ' // Do we have a second value? - - If CanGet(Here) AndAlso IsColon(Peek(Here)) Then + Dim c2 As Char + If TryPeek(Here, c2) AndAlso IsColon(c2) Then ' // Yes. HaveSecondValue = True Here += 1 @@ -2359,24 +2038,22 @@ FullWidthRepeat2: ' // Check AM/PM - If CanGet(Here) Then - If Peek(Here) = "A"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_A OrElse - Peek(Here) = "a"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_A Then - + If TryPeek(Here, c1) Then + If c1.IsAnyOf("A"c, FULLWIDTH_LATIN_CAPITAL_LETTER_A, + "a"c, FULLWIDTH_LATIN_SMALL_LETTER_A) Then HaveAM = True Here += 1 - ElseIf Peek(Here) = "P"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_P OrElse - Peek(Here) = "p"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_P Then - + ElseIf c1.IsAnyOf("P"c, FULLWIDTH_LATIN_CAPITAL_LETTER_P, + "p"c, FULLWIDTH_LATIN_SMALL_LETTER_P) Then HavePM = True Here += 1 End If - - If CanGet(Here) AndAlso (HaveAM OrElse HavePM) Then - If Peek(Here) = "M"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_M OrElse - Peek(Here) = "m"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_M Then + Dim c2 As Char + If TryPeek(Here, c2) AndAlso (HaveAM OrElse HavePM) Then + If c2.IsAnyOf("M"c, FULLWIDTH_LATIN_CAPITAL_LETTER_M, + "m"c, FULLWIDTH_LATIN_SMALL_LETTER_M) Then Here = GetWhitespaceLength(Here + 1) @@ -2393,7 +2070,7 @@ FullWidthRepeat2: End If End If - If Not CanGet(Here) OrElse Not IsHash(Peek(Here)) Then + If Not TryPeek(Here, c) OrElse Not IsHash(c) Then GoTo baddate End If @@ -2498,16 +2175,12 @@ FullWidthRepeat2: baddate: ' // If we can find a closing #, then assume it's a malformed date, ' // otherwise, it's not a date - - While CanGet(Here) - Dim ch As Char = Peek(Here) - If IsHash(ch) OrElse IsNewLine(ch) Then - Exit While - End If + Dim ch As Char + While TryPeek(Here, ch) AndAlso Not (IsHash(ch) OrElse IsNewLine(ch)) Here += 1 End While - If Not CanGet(Here) OrElse IsNewLine(Peek(Here)) Then + If Not TryPeek(Here, ch) OrElse IsNewLine(ch) Then ' // No closing # Return Nothing Else @@ -2522,44 +2195,35 @@ baddate: Debug.Assert(IsDoubleQuote(Peek)) Dim length As Integer = 1 - Dim ch As Char + Dim ch, cx As Char Dim followingTrivia As SyntaxList(Of VisualBasicSyntaxNode) ' // Check for a Char literal, which can be of the form: ' // """"c or ""c - If CanGet(3) AndAlso IsDoubleQuote(Peek(2)) Then + If TryPeek(3, ch) AndAlso IsDoubleQuote(Peek(2)) Then If IsDoubleQuote(Peek(1)) Then - If IsDoubleQuote(Peek(3)) AndAlso - CanGet(4) AndAlso - IsLetterC(Peek(4)) Then - + If IsDoubleQuote(ch) AndAlso TryPeek(4, cx) AndAlso IsLetterC(cx) Then ' // Double-quote Char literal: """"c Return MakeCharacterLiteralToken(precedingTrivia, """"c, 5) End If - ElseIf IsLetterC(Peek(3)) Then + ElseIf IsLetterC(ch) Then ' // Char literal. "x"c Return MakeCharacterLiteralToken(precedingTrivia, Peek(1), 4) End If End If - If CanGet(2) AndAlso - IsDoubleQuote(Peek(1)) AndAlso - IsLetterC(Peek(2)) Then - + If TryPeek(2, ch) AndAlso IsDoubleQuote(Peek(1)) AndAlso IsLetterC(ch) Then ' // Error. ""c is not a legal char constant Return MakeBadToken(precedingTrivia, 3, ERRID.ERR_IllegalCharConstant) End If Dim scratch = GetScratch() - While CanGet(length) - ch = Peek(length) + While TryPeek(length, ch) If IsDoubleQuote(ch) Then - If CanGet(length + 1) Then - ch = Peek(length + 1) - + If TryPeek(length + 1, ch) Then If IsDoubleQuote(ch) Then ' // An escaped double quote scratch.Append(""""c) @@ -2569,7 +2233,6 @@ baddate: ' // The end of the char literal. If IsLetterC(ch) Then ' // Error. "aad"c is not a legal char constant - ' // +2 to include both " and c in the token span scratch.Clear() Return MakeBadToken(precedingTrivia, length + 2, ERRID.ERR_IllegalCharConstant) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index eb30658604ee0..b46d63d2f1fd2 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -95,6 +95,36 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return p End Function + Friend Function TryPeek(at As Integer, ByRef ch As Char) As Boolean + ' CanGet(at) + Debug.Assert(_lineBufferOffset + at >= 0) + Debug.Assert(at >= -MaxCharsLookBehind) + If _lineBufferOffset + at >= _bufferLen Then Return False + ' Peek(at) + at += _lineBufferOffset + Dim page = _curPage + ch = page._arr(at And s_PAGE_MASK) + If page._pageStart <> (at And s_NOT_PAGE_MASK) Then + page = GetPage(at) + ch = page._arr(at And s_PAGE_MASK) + End If + Return True + End Function + + Friend Function TryPeek(ByRef ch As Char) As Boolean + ' CanGet() + If _lineBufferOffset >= _bufferLen Then Return False + ' Peek() + Dim page = _curPage + Dim position = _lineBufferOffset + ch = page._arr(position And s_PAGE_MASK) + If page._pageStart <> (position And s_NOT_PAGE_MASK) Then + page = GetPage(position) + ch = page._arr(position And s_PAGE_MASK) + End If + Return True + End Function + ' PERF CRITICAL Private Function Peek(skip As Integer) As Char Debug.Assert(CanGet(skip)) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb index 40a522c7dbfdd..24be6bd253ef0 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb @@ -19,26 +19,25 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim leadingTriviaLength = GetWhitespaceLength(0) Dim offset = leadingTriviaLength Dim length As Integer - - If Not CanGet(offset) Then + Dim c As Char + If Not TryPeek(offset, c) Then Return MakeEndOfInterpolatedStringToken() End If - Dim c = Peek(offset) - ' This should only ever happen for $" or } - Debug.Assert(leadingTriviaLength = 0 OrElse c = "$"c OrElse c = FULLWIDTH_DOLLAR_SIGN OrElse IsRightCurlyBracket(c)) + Debug.Assert(leadingTriviaLength = 0 OrElse c.IsAnyOf("$"c, FULLWIDTH_DOLLAR_SIGN) OrElse IsRightCurlyBracket(c)) ' Another } may follow the close brace of an interpolation if the interpolation lacked a format clause. ' This is because the normal escaping rules only apply when parsing the format string. Debug.Assert(Not CanGet(1) OrElse Peek(offset + 1) <> c OrElse IsRightCurlyBracket(c), "Escape sequence not detected.") Dim scanTrailingTrivia As Boolean + Dim c1 As Char Select Case c Case "$"c, FULLWIDTH_DOLLAR_SIGN - If CanGet(offset + 1) AndAlso IsDoubleQuote(Peek(offset + 1)) Then + If TryPeek(offset + 1, c1) AndAlso IsDoubleQuote(c1) Then kind = SyntaxKind.DollarSignDoubleQuoteToken length = 2 scanTrailingTrivia = False ' Trailing whitespace should be scanned as interpolated string text. @@ -112,22 +111,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function IsInterpolatedStringPunctuation(Optional offset As Integer = 0) As Boolean - If Not CanGet(offset) Then Return False - - Dim c = Peek(offset) - + Dim c, c1 As Char + If Not TryPeek(offset, c) Then Return False + Dim ok = TryPeek(offset + 1, c1) If IsLeftCurlyBracket(c) Then - Return Not CanGet(offset + 1) OrElse Not IsLeftCurlyBracket(Peek(offset + 1)) + Return Not ok OrElse Not IsLeftCurlyBracket(c1) ElseIf IsRightCurlyBracket(c) Then - Return Not CanGet(offset + 1) OrElse Not IsRightCurlyBracket(Peek(offset + 1)) + Return Not ok OrElse Not IsRightCurlyBracket(c1) ElseIf IsDoubleQuote(c) 'A subtle difference between this case and the one above. ' In both interpolated and literal strings the two quote characters used in an escape sequence don't have to match. ' It's enough that the next character is *a* quote char. It doesn't have to be the same quote. ' If we want to preserve consistency the quotes need to be special cased. - Return Not CanGet(offset + 1) OrElse Not IsDoubleQuote(Peek(offset + 1)) + Return Not ok OrElse Not IsDoubleQuote(c1) Else Return False @@ -140,17 +138,16 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim offset = 0 Dim pendingWhitespace = 0 Dim valueBuilder = GetScratch() - - Do While CanGet(offset) - - Dim c = Peek(offset) + Dim c, c1 As Char + Do While TryPeek(offset, c) ' Any combination of fullwidth and ASCII curly braces of the same direction is an escaping sequence for the corresponding ASCII curly brace. ' We insert that curly brace doubled and because this is the escaping sequence understood by String.Format, that will be replaced by a single brace. ' This is deliberate design and it aligns with existing rules for double quote escaping in strings. + Dim ok = TryPeek(offset+1, c1) If IsLeftCurlyBracket(c) Then - If CanGet(offset + 1) AndAlso IsLeftCurlyBracket(Peek(offset + 1)) Then + If ok AndAlso IsLeftCurlyBracket(c1) Then ' This is an escape sequence. valueBuilder.Append("{{") @@ -164,7 +161,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ElseIf IsRightCurlyBracket(c) Then - If CanGet(offset + 1) AndAlso IsRightCurlyBracket(Peek(offset + 1)) Then + If ok AndAlso IsRightCurlyBracket(c1) Then ' This is an escape sequence. valueBuilder.Append("}}") @@ -178,7 +175,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ElseIf IsDoubleQuote(c) - If CanGet(offset + 1) AndAlso IsDoubleQuote(Peek(offset + 1)) Then + If ok AndAlso IsDoubleQuote(c1) Then ' This is a VB double quote escape. Oddly enough this logic allows mixing and matching of ' smart and dumb double quotes in any order. Regardless we always emit as a standard double quote. ' This is consistent with their handling in string literals. diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb index ae1cc8929a00f..e296a1ef7c6af 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb @@ -16,15 +16,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanXmlTrivia(c As Char) As SyntaxList(Of VisualBasicSyntaxNode) Debug.Assert(Not IsScanningXmlDoc) - Debug.Assert(c = CARRIAGE_RETURN OrElse c = LINE_FEED OrElse c = " "c OrElse c = CHARACTER_TABULATION) + Debug.Assert(c.IsAnyOf( CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION)) Dim builder = _triviaListPool.Allocate Dim len = 0 Do - If c = " "c OrElse c = CHARACTER_TABULATION Then + If c.IsAnyOf(" "c, CHARACTER_TABULATION) Then len += 1 - ElseIf c = CARRIAGE_RETURN OrElse c = LINE_FEED Then + ElseIf c.IsAnyOf(CARRIAGE_RETURN, LINE_FEED) Then If len > 0 Then builder.Add(MakeWhiteSpaceTrivia(GetText(len))) len = 0 @@ -34,10 +34,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do End If - If Not CanGet(len) Then + If Not TryPeek(len, c) Then Exit Do End If - c = Peek(len) Loop If len > 0 Then @@ -69,8 +68,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing - While CanGet() - Dim c As Char = Peek() + Dim c As Char + While TryPeek(c) Select Case (c) ' // Whitespace @@ -93,7 +92,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax leadingTrivia = ScanXmlTrivia(c) Case "/"c - If CanGet(1) AndAlso Peek(1) = ">" Then + If NextIs(1,">"c) Then Return XmlMakeEndEmptyElementToken(leadingTrivia) End If Return XmlMakeDivToken(leadingTrivia) @@ -113,12 +112,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeDoubleQuoteToken(leadingTrivia, c, isOpening:=True) Case "<"c - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If TryPeek(1, ch) Then + Dim c2 As Char Select Case ch Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + If TryPeek(2,c2) Then + Select Case c2 Case "-"c If NextIs(3,"-"c) Then Return XmlMakeBeginCommentToken(leadingTrivia, s_scanNoTriviaFunc) @@ -206,14 +206,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '// NL WS* ' '// Example ' This is a comment Private Function ScanXmlForPossibleStatement(state As ScannerState) As Boolean - If Not CanGet() Then + Dim c As Char + If Not TryPeek(c) Then Return False End If Dim token As SyntaxToken Dim possibleStatement As Boolean = False Dim offsets = CreateOffsetRestorePoint() - Dim c As Char = Peek() Select Case c Case "#"c, @@ -238,8 +238,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If name.PossibleKeywordKind <> SyntaxKind.XmlNameToken Then leadingTrivia = ScanSingleLineTrivia() c = Peek() - possibleStatement = - c = "("c OrElse c = FULLWIDTH_LEFT_PARENTHESIS + possibleStatement = c.IsAnyOf("("c, FULLWIDTH_LEFT_PARENTHESIS) End If End If @@ -308,8 +307,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim scratch = GetScratch() - While CanGet(Here) - Dim c As Char = Peek(Here) + Dim c As Char + While TryPeek(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -341,12 +340,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Debug.Assert(Here = 0) - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If TryPeek(1, ch) Then Select Case ch Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + Dim c2 As Char + If TryPeek(2, c2) Then + Select Case c2 Case "-"c If NextIs(3,"-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) @@ -489,9 +489,9 @@ ScanChars: End If Dim Here = 0 - While CanGet(Here) - Dim c As Char = Peek(Here) - Select Case (c) + Dim c As Char + While TryPeek(Here, c) + Select Case c Case CARRIAGE_RETURN, LINE_FEED Return XmlMakeCommentToken(precedingTrivia, Here + LengthOfLineBreak(c, Here)) @@ -507,9 +507,7 @@ ScanChars: Return XmlMakeCommentToken(precedingTrivia, Here) End If - If CanGet(Here + 2) Then - - c = Peek(Here + 2) + If TryPeek(Here + 2, c) Then Here += 2 ' // if > is not found then this is an error. Return the -- string @@ -579,11 +577,9 @@ ScanChars: Dim scratch = GetScratch() Dim Here = 0 - - While CanGet(Here) - Dim c As Char = Peek(Here) - Select Case (c) - + Dim c As Char + While TryPeek(Here, c) + Select Case c Case CARRIAGE_RETURN, LINE_FEED Here = SkipLineBreak(c, Here) scratch.Append(LINE_FEED) @@ -643,11 +639,10 @@ ScanChars: Dim precedingTrivia = _triviaListPool.Allocate(Of VisualBasicSyntaxNode)() Dim result As SyntaxToken - - If state = ScannerState.StartProcessingInstruction AndAlso CanGet() Then + Dim c As Char + If state = ScannerState.StartProcessingInstruction AndAlso TryPeek(c) Then ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ - Dim c = Peek() Select Case c Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION Dim wsTrivia = ScanXmlTrivia(c) @@ -656,8 +651,7 @@ ScanChars: End If Dim Here = 0 - While CanGet(Here) - Dim c As Char = Peek(Here) + While TryPeek(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -715,9 +709,8 @@ CleanUp: ' // Misc ::= Comment | PI | S Dim precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing - While CanGet() - Dim c As Char = Peek() - + Dim c As Char + While TryPeek(c) Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -727,17 +720,17 @@ CleanUp: precedingTrivia = ScanXmlTrivia(c) Case "<"c - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If TryPeek(1, ch) Then Select Case ch Case "!"c - If NextAre(2,"--") Then + If NextAre(2, "--") Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) - ElseIf NextAre(2,"DOCTYPE") Then + ElseIf NextAre(2, "DOCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If Case "%"c - If NextIs(2,"="c) Then + If NextIs(2, "="c) Then Return XmlMakeBeginEmbeddedToken(precedingTrivia) End If Case "?"c @@ -781,7 +774,8 @@ CleanUp: End Function Friend Function ScanXmlStringUnQuoted() As SyntaxToken - If Not CanGet() Then + Dim c As Char + If Not TryPeek(c) Then Return MakeEofToken() End If @@ -791,11 +785,8 @@ CleanUp: Dim Here = 0 Dim scratch = GetScratch() - While CanGet(Here) - Dim c As Char = Peek(Here) - + While TryPeek(Here,c) Select Case (c) - Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION If Here > 0 Then Return XmlMakeAttributeDataToken(Nothing, Here, scratch) @@ -883,9 +874,8 @@ ScanChars: Dim Here = 0 Dim scratch = GetScratch() - - While CanGet(Here) - Dim c As Char = Peek(Here) + Dim c As Char + While TryPeek(Here, c) If c = terminatingChar Or c = altTerminatingChar Then If Here > 0 Then result = XmlMakeAttributeDataToken(precedingTrivia, Here, scratch) @@ -986,10 +976,8 @@ CleanUp: Debug.Assert(Here >= 0) Debug.Assert(CanGet(Here)) Debug.Assert(Peek(Here) = c1) - - If IsHighSurrogate(c1) AndAlso CanGet(Here + 1) Then - Dim c2 = Peek(Here + 1) - + Dim c2 As Char + If IsHighSurrogate(c1) AndAlso TryPeek(Here + 1, c2) Then If IsLowSurrogate(c2) Then Return New XmlCharResult(c1, c2) End If @@ -1028,9 +1016,9 @@ CleanUp: Private Function ScanXmlChar(Here As Integer) As XmlCharResult Debug.Assert(Here >= 0) - Debug.Assert(CanGet(Here)) - - Dim c = Peek(Here) + Dim c As Char + Dim ok =TryPeek(Here, c) + Debug.Assert(ok) If Not isValidUtf16(c) Then Return Nothing @@ -1061,9 +1049,8 @@ CleanUp: 'TODO - Fix ScanXmlNCName to conform to XML spec instead of old loose scanning. - While CanGet(Here) - Dim c As Char = Peek(Here) - + Dim c As Char + While TryPeek(Here, c) Select Case (c) Case ":"c, " "c, CHARACTER_TABULATION, LINE_FEED, CARRIAGE_RETURN, @@ -1125,14 +1112,11 @@ CreateNCNameToken: End Function Private Function ScanXmlReference(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As XmlTextTokenSyntax - Debug.Assert(CanGet) - Debug.Assert(Peek() = "&"c) - + Debug.Assert( NextIs(0,"&"c)) + Dim c,c1 As Char ' skip 1 char for "&" - If CanGet(1) Then - Dim c As Char = Peek(1) - - Select Case (c) + If TryPeek(1, c) Then + Select Case (c) Case "#"c Dim Here = 2 ' skip "&#" Dim result = ScanXmlCharRef(Here) @@ -1145,7 +1129,7 @@ CreateNCNameToken: value = Intern({result.Char1, result.Char2}) End If - If CanGet(Here) AndAlso Peek(Here) = ";"c Then + If NextIs(Here, ";"c) Then Return XmlMakeEntityLiteralToken(precedingTrivia, Here + 1, value) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, Here, value) @@ -1157,8 +1141,8 @@ CreateNCNameToken: Case "a"c ' // & ' // ' - If CanGet(4) AndAlso NextAre(2,"mp") Then - If Peek(4) = ";"c Then + If TryPeek(4, c1) AndAlso NextAre(2,"mp") Then + If c1 = ";"c Then Return XmlMakeAmpLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 4, "&") @@ -1166,9 +1150,8 @@ CreateNCNameToken: Return DirectCast(noSemicolon.SetDiagnostics({noSemicolonError}), XmlTextTokenSyntax) End If - ElseIf CanGet(5) AndAlso NextAre(2,"pos") Then - - If Peek(5) = ";"c Then + ElseIf TryPeek(5, c1) AndAlso NextAre(2,"pos") Then + If c1 = ";"c Then Return XmlMakeAposLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 5, "'") @@ -1179,8 +1162,8 @@ CreateNCNameToken: Case "l"c ' // < - If CanGet(3) AndAlso NextIs(2,"t"c) Then - If Peek(3) = ";"c Then + If TryPeek(3, c1) AndAlso NextIs(2,"t"c) Then + If c1 = ";"c Then Return XmlMakeLtLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 3, "<") @@ -1191,8 +1174,8 @@ CreateNCNameToken: Case "g"c ' // > - If CanGet(3) AndAlso NextIs(2,"t"c) Then - If Peek(3) = ";"c Then + If TryPeek(3, c1) AndAlso NextIs(2,"t"c) Then + If c1 = ";"c Then Return XmlMakeGtLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 3, ">") @@ -1203,8 +1186,8 @@ CreateNCNameToken: Case "q"c ' // " - If CanGet(5) AndAlso NextAre(2,"uot") Then - If Peek(5) = ";"c Then + If TryPeek(5, c1) AndAlso NextAre(2,"uot") Then + If c1 = ";"c Then Return XmlMakeQuotLiteralToken(precedingTrivia) Else Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 5, """") @@ -1224,21 +1207,18 @@ CreateNCNameToken: Private Function ScanXmlCharRef(ByRef index As Integer) As XmlCharResult Debug.Assert(index >= 0) - - If Not CanGet(index) Then + Dim ch As Char + If Not TryPeek(index, ch) Then Return Nothing End If ' cannot reuse Scratch as this can be used in a nested call. Dim charRefSb As New StringBuilder Dim Here = index - - Dim ch = Peek(Here) If ch = "x"c Then Here += 1 - While CanGet(Here) - ch = Peek(Here) + While TryPeek(Here, ch) If XmlCharType.IsHexDigit(ch) Then charRefSb.Append(ch) Else @@ -1254,13 +1234,9 @@ CreateNCNameToken: Return result End If Else - While CanGet(Here) - ch = Peek(Here) - If XmlCharType.IsDigit(ch) Then - charRefSb.Append(ch) - Else - Exit While - End If + + While TryPeek(Here, ch) AndAlso XmlCharType.IsDigit(ch) + charRefSb.Append(ch) Here += 1 End While If charRefSb.Length > 0 Then diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index fd7c3deb6af9b..fae9ef70f17b4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -50,7 +50,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function TryScanXmlDocComment(tList As SyntaxListBuilder) As Boolean Debug.Assert(IsAtNewLine) ' leading whitespace until we see ''' should be regular whitespace - If CanGet() AndAlso IsWhitespace(Peek()) Then + Dim c As Char + If TryPeek(c) AndAlso IsWhitespace(c) Then Dim ws = ScanWhitespace() tList.Add(ws) End If @@ -146,14 +147,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' lexes (ws)''' Private Function TrySkipXmlDocMarker(ByRef len As Integer) As Boolean + Dim c As Char Dim Here = len - While CanGet(Here) - Dim c = Peek(Here) - If IsWhitespace(c) Then - Here += 1 - Else - Exit While - End If + While TryPeek(Here, c) AndAlso IsWhitespace(c) + Here += 1 End While If StartsXmlDoc(Here) Then @@ -185,7 +182,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim len = 0 Do - If c = " "c OrElse c = CHARACTER_TABULATION Then + If c.IsAnyOf(" "c, CHARACTER_TABULATION) Then len += 1 ElseIf IsNewLine(c) Then @@ -239,9 +236,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim Here As Integer = 0 Dim scratch = GetScratch() - - While CanGet(Here) - Dim c As Char = Peek(Here) + Dim c As Char + While TryPeek(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -281,12 +277,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Debug.Assert(Here = 0) - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch As Char + If TryPeek(1, ch) Then Select Case ch Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + Dim c2 As Char + If TryPeek(2, c2) Then + Select Case c2 Case "-"c If NextIs(3,"-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) @@ -368,10 +365,10 @@ ScanChars: precedingTrivia.Add(xDocTrivia) End If - If state = ScannerState.StartProcessingInstruction AndAlso CanGet() Then + Dim c As Char + If state = ScannerState.StartProcessingInstruction AndAlso TryPeek(c) Then ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ - Dim c = Peek() Select Case c Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION Dim offsets = CreateOffsetRestorePoint() @@ -385,8 +382,7 @@ ScanChars: End If Dim Here = 0 - While CanGet(Here) - Dim c As Char = Peek(Here) + While TryPeek(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -458,7 +454,8 @@ CleanUp: precedingTrivia = New SyntaxList(Of VisualBasicSyntaxNode)(xDocTrivia) End If - While CanGet() + Dim c As Char + While TryPeek(c) If Not precedingTrivia.Any AndAlso IsAtNewLine() AndAlso Not Me._doNotRequireXmlDocCommentPrefix Then ' this would indicate that we looked at Trivia, but did not find ' XmlDoc prefix (or we would not be at the line start) @@ -466,8 +463,6 @@ CleanUp: Return MakeEofToken(precedingTrivia) End If - Dim c As Char = Peek() - Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -503,12 +498,12 @@ CleanUp: Return XmlMakeDoubleQuoteToken(precedingTrivia, c, isOpening:=True) Case "<"c - If CanGet(1) Then - Dim ch As Char = Peek(1) + Dim ch,c2 As Char + If TryPeek(1, ch) Then Select Case ch Case "!"c - If CanGet(2) Then - Select Case (Peek(2)) + If TryPeek(2, c2) Then + Select Case c2 Case "-"c If NextIs(3,"-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb index a384df6737af6..c98efa4c657e3 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb @@ -317,11 +317,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function XmlMakeEndEmbeddedToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax - Debug.Assert(Peek() = "%"c OrElse Peek() = FULLWIDTH_PERCENT_SIGN) + Dim c = Peek + Debug.Assert(c.IsAnyOf("%"c, FULLWIDTH_PERCENT_SIGN)) Debug.Assert(Peek(1) = ">"c) Dim spelling As String - If Peek() = "%"c Then + If c = "%"c Then AdvanceChar(2) spelling = "%>" Else From 6fec504c5c8c3c0ca0a0039f627b5e4062ccbe1c Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Wed, 1 Apr 2015 21:27:12 +0100 Subject: [PATCH 4/9] [VB: Scanner] PR#1463 Changes via Comments * Make some code DEBUG only * Change single line IF Then into Full Version --- .../VisualBasic/Portable/Scanner/Scanner.vb | 44 ++++++++++--------- .../Portable/Scanner/ScannerBuffer.vb | 5 ++- .../Scanner/ScannerInterpolatedString.vb | 14 +++--- .../Portable/Scanner/ScannerXml.vb | 6 +-- .../Portable/Scanner/XmlDocComments.vb | 2 +- 5 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index 26ab2e2557adb..98ba6e2ac96e6 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -485,11 +485,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function LengthOfLineBreak(StartCharacter As Char, Optional here As Integer = 0) As Integer +#If DEBUG Then Dim c As Char Dim res = TryPeek(here, c) Debug.Assert(res) Debug.Assert(IsNewLine(StartCharacter)) - Debug.Assert(StartCharacter = Peek(here)) + Debug.Assert(StartCharacter = c) +#End If If StartCharacter = CARRIAGE_RETURN AndAlso NextIs(here + 1, LINE_FEED) Then Return 2 End If @@ -900,12 +902,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Sub EatWhitespace() Dim c As Char +#If DEBUG Then Dim ok = TryPeek(c) Debug.Assert(ok) Debug.Assert(IsWhitespace(c)) - +#End If AdvanceChar() - ' eat until linebreak or nonwhitespace While TryPeek(c) AndAlso IsWhitespace(c) AdvanceChar() @@ -986,8 +988,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Case CARRIAGE_RETURN, LINE_FEED Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) Case NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR - If fullWidth Then Exit Select - Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) + If Not fullWidth Then + Return ScanNewlineAsStatementTerminator(ch, precedingTrivia) + End If Case " "c, CHARACTER_TABULATION, "'"c Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch))) Return Nothing ' trivia cannot start a token @@ -1079,27 +1082,24 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Case "A"c If NextAre(1, "s ") Then ' TODO: do we allow widechars in keywords? - Dim spelling = "As" - If fullWidth Then spelling = GetText(2) Else AdvanceChar(2) - Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia) + AdvanceChar(2) + Return MakeKeyword(SyntaxKind.AsKeyword, "As", precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If Case "E"c If NextAre(1, "nd ") Then ' TODO: do we allow widechars in keywords? - Dim spelling = "End" - If fullWidth Then spelling = GetText(3) Else AdvanceChar(3) - Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia) + AdvanceChar(3) + Return MakeKeyword(SyntaxKind.EndKeyword, "End", precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If Case "I"c If NextAre(1, "f ") Then ' TODO: do we allow widechars in keywords? - Dim spelling = "If" - If fullWidth Then spelling = GetText(2) Else AdvanceChar(2) - Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia) + AdvanceChar(2) + Return MakeKeyword(SyntaxKind.IfKeyword, "If", precedingTrivia) Else Return ScanIdentifierOrKeyword(precedingTrivia) End If @@ -1129,8 +1129,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc) End If Case "$"c, FULLWIDTH_DOLLAR_SIGN - If fullWidth Then Exit Select - If TryPeek(1, c) AndAlso IsDoubleQuote(c) Then + If Not fullWidth AndAlso TryPeek(1, c) AndAlso IsDoubleQuote(c) Then Return MakePunctuationToken(precedingTrivia, 2, SyntaxKind.DollarSignDoubleQuoteToken) End If End Select @@ -1191,9 +1190,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanRightAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean) As SyntaxToken Dim c As Char +#If DEBUG Then Dim ok = TryPeek(c) Debug.Assert(ok) ' > Debug.Assert(c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN)) +#End If + Dim length As Integer = 1 ' // Allow whitespace between the characters of a two-character token. @@ -1217,9 +1219,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanLeftAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean, scanTrailingTrivia As ScanTriviaFunc) As SyntaxToken Dim c As Char +#If DEBUG Then Dim ok = TryPeek(c) Debug.Assert(ok) ' < Debug.Assert(c.IsAnyOf("<"c, FULLWIDTH_LESS_THAN_SIGN)) +#End If Dim length As Integer = 1 ' Check for XML tokens @@ -1251,8 +1255,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - - If TryPeek(length, c) Then If c.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 @@ -1309,7 +1311,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanIdentifierOrKeyword(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim ch As Char - Dim ok = TryPeek(ch) + Dim ok = TryPeek(ch) ' Non DEBUG usage see:= If IsConnectorPunctuation(ch) + Debug.Assert(ok) Debug.Assert(IsIdentifierStartCharacter(ch)) Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller @@ -1422,9 +1425,11 @@ FullWidthRepeat: Private Function ScanBracketedIdentifier(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim ch As Char +#If DEBUG Then Dim ok = TryPeek(ch) Debug.Assert(ok) ' [ Debug.Assert(ch.IsAnyOf("["c, FULLWIDTH_LEFT_SQUARE_BRACKET)) +#End If Dim IdStart As Integer = 1 Dim Here As Integer = IdStart @@ -1914,7 +1919,6 @@ FullWidthRepeat2: ' // The first thing has to be an integer, although it's not clear what it is yet If Not ScanIntLiteral(FirstValue, Here) Then Return Nothing - End If ' // If we see a /, then it's a date diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index b46d63d2f1fd2..fa06b5c9f7dca 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -4,6 +4,7 @@ ' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- +Imports System.Runtime.InteropServices Imports System.Text Imports Microsoft.CodeAnalysis.Text Imports Microsoft.CodeAnalysis.VisualBasic.Symbols @@ -95,7 +96,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return p End Function - Friend Function TryPeek(at As Integer, ByRef ch As Char) As Boolean + Friend Function TryPeek(at As Integer, ByRef ch As Char) As Boolean ' CanGet(at) Debug.Assert(_lineBufferOffset + at >= 0) Debug.Assert(at >= -MaxCharsLookBehind) @@ -111,7 +112,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return True End Function - Friend Function TryPeek(ByRef ch As Char) As Boolean + Friend Function TryPeek( ByRef ch As Char) As Boolean ' CanGet() If _lineBufferOffset >= _bufferLen Then Return False ' Peek() diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb index 24be6bd253ef0..b96fe897fb62b 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb @@ -113,19 +113,18 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function IsInterpolatedStringPunctuation(Optional offset As Integer = 0) As Boolean Dim c, c1 As Char If Not TryPeek(offset, c) Then Return False - Dim ok = TryPeek(offset + 1, c1) If IsLeftCurlyBracket(c) Then - Return Not ok OrElse Not IsLeftCurlyBracket(c1) + Return Not TryPeek(offset + 1, c1) OrElse Not IsLeftCurlyBracket(c1) ElseIf IsRightCurlyBracket(c) Then - Return Not ok OrElse Not IsRightCurlyBracket(c1) + Return Not TryPeek(offset + 1, c1) OrElse Not IsRightCurlyBracket(c1) ElseIf IsDoubleQuote(c) 'A subtle difference between this case and the one above. ' In both interpolated and literal strings the two quote characters used in an escape sequence don't have to match. ' It's enough that the next character is *a* quote char. It doesn't have to be the same quote. ' If we want to preserve consistency the quotes need to be special cased. - Return Not ok OrElse Not IsDoubleQuote(c1) + Return Not TryPeek(offset + 1, c1) OrElse Not IsDoubleQuote(c1) Else Return False @@ -144,10 +143,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' Any combination of fullwidth and ASCII curly braces of the same direction is an escaping sequence for the corresponding ASCII curly brace. ' We insert that curly brace doubled and because this is the escaping sequence understood by String.Format, that will be replaced by a single brace. ' This is deliberate design and it aligns with existing rules for double quote escaping in strings. - Dim ok = TryPeek(offset+1, c1) If IsLeftCurlyBracket(c) Then - If ok AndAlso IsLeftCurlyBracket(c1) Then + If TryPeek(offset + 1, c1) AndAlso IsLeftCurlyBracket(c1) Then ' This is an escape sequence. valueBuilder.Append("{{") @@ -161,7 +159,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ElseIf IsRightCurlyBracket(c) Then - If ok AndAlso IsRightCurlyBracket(c1) Then + If TryPeek(offset + 1, c1) AndAlso IsRightCurlyBracket(c1) Then ' This is an escape sequence. valueBuilder.Append("}}") @@ -175,7 +173,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ElseIf IsDoubleQuote(c) - If ok AndAlso IsDoubleQuote(c1) Then + If TryPeek(offset + 1, c1) AndAlso IsDoubleQuote(c1) Then ' This is a VB double quote escape. Oddly enough this logic allows mixing and matching of ' smart and dumb double quotes in any order. Regardless we always emit as a standard double quote. ' This is consistent with their handling in string literals. diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb index e296a1ef7c6af..f4e0a630d0e47 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb @@ -774,14 +774,14 @@ CleanUp: End Function Friend Function ScanXmlStringUnQuoted() As SyntaxToken - Dim c As Char - If Not TryPeek(c) Then + If Not CanGet() Then Return MakeEofToken() End If ' This can never happen as this token cannot cross lines. Debug.Assert(Not (IsScanningXmlDoc AndAlso IsAtNewLine())) + Dim c As Char Dim Here = 0 Dim scratch = GetScratch() @@ -1113,7 +1113,7 @@ CreateNCNameToken: Private Function ScanXmlReference(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As XmlTextTokenSyntax Debug.Assert( NextIs(0,"&"c)) - Dim c,c1 As Char + Dim c, c1 As Char ' skip 1 char for "&" If TryPeek(1, c) Then Select Case (c) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index fae9ef70f17b4..5a0024bdcab22 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -498,7 +498,7 @@ CleanUp: Return XmlMakeDoubleQuoteToken(precedingTrivia, c, isOpening:=True) Case "<"c - Dim ch,c2 As Char + Dim ch, c2 As Char If TryPeek(1, ch) Then Select Case ch Case "!"c From b401f40b0510c2df7c60879a92d164d7e1680833 Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Sun, 5 Apr 2015 03:41:57 +0100 Subject: [PATCH 5/9] [VB: Scanner] Comments Suggested Changes * Renamed the following methods TryPeek --> Peep IsAnyOf --> IsFrom IsNoneOf --> IsntFrom * Remove some temporary variables via the useage of Peep --- .../Portable/Scanner/CharacterInfo.vb | 12 +- .../Portable/Scanner/KeywordTable.vb | 3 +- .../VisualBasic/Portable/Scanner/Scanner.vb | 232 ++++++++---------- .../Portable/Scanner/ScannerBuffer.vb | 30 ++- .../Scanner/ScannerInterpolatedString.vb | 30 +-- .../Portable/Scanner/ScannerXml.vb | 72 +++--- .../Portable/Scanner/XmlDocComments.vb | 51 ++-- .../Portable/Scanner/XmlTokenFactories.vb | 2 +- 8 files changed, 212 insertions(+), 220 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb index 32fa7222eac95..575ca17d7839a 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb @@ -491,32 +491,32 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Friend Module CharExts - Friend Function IsAnyOf(c As Char, c0 As Char, c1 As Char) As Boolean + Friend Function IsFrom(c As Char, c0 As Char, c1 As Char) As Boolean Return (c = c0) OrElse (c = c1) End Function - Friend Function IsAnyOf(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Friend Function IsFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean Return (c = c0) OrElse (c = c1) OrElse (c = c2) End Function - Friend Function IsAnyOf(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Friend Function IsFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean Return (c = c0) OrElse (c = c1) OrElse (c = c2) OrElse (c = c3) End Function - Friend Function IsNoneOf(c As Char, c0 As Char, c1 As Char) As Boolean + Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) End Function - Friend Function IsNoneOf(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) End Function - Friend Function IsNoneOf(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) AndAlso (c <> c3) End Function End Module diff --git a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb index 7d896cca19342..c45ede3b05bb2 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb @@ -304,9 +304,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax Private Shared Function EnsureHalfWidth(s As String) As String Dim result As Char() = Nothing - Dim ch As Char For i As Integer = 0 To s.Length - 1 - ch = s(i) + Dim ch As Char = s(i) If SyntaxFacts.IsFullWidth(ch) Then ch = SyntaxFacts.MakeHalfWidth(ch) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index 98ba6e2ac96e6..08f3e6540bd70 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -226,7 +226,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanNextCharAsToken(leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim token As SyntaxToken Dim c As Char - If Not TryPeek(c) Then + If Not Peep(c) Then token = MakeEofToken(leadingTrivia) Else _badTokenCount += 1 @@ -240,7 +240,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' If we get too many characters that we cannot make sense of, absorb the rest of the input. token = MakeBadToken(leadingTrivia, RemainingLength(), ERRID.ERR_IllegalChar) End If - End If Return token @@ -267,7 +266,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim condLineStart = _lineBufferOffset Dim c As Char - While TryPeek(c) + While Peep(c) Select Case (c) @@ -321,9 +320,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Sub EatThroughLine() - While CanGet() - Dim c As Char = Peek() - If IsNewLine(c) Then + Dim c As Char + While Peep(c) + If IsNewLine(c) Then EatThroughLineBreak(c) Return Else @@ -487,7 +486,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function LengthOfLineBreak(StartCharacter As Char, Optional here As Integer = 0) As Integer #If DEBUG Then Dim c As Char - Dim res = TryPeek(here, c) + Dim res = Peep(here, c) Debug.Assert(res) Debug.Assert(IsNewLine(StartCharacter)) Debug.Assert(StartCharacter = c) @@ -534,12 +533,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanLineContinuation(tList As SyntaxListBuilder) As Boolean Dim ch As Char - If Not TryPeek(ch) OrElse Not IsAfterWhitespace() OrElse Not IsUnderscore(ch) Then + If Not Peep(ch) OrElse Not IsAfterWhitespace() OrElse Not IsUnderscore(ch) Then Return False End If Dim Here = 1 - While TryPeek(Here, ch) AndAlso IsWhitespace(ch) + While Peep(Here, ch) AndAlso IsWhitespace(ch) Here += 1 End While @@ -566,7 +565,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' implicit line continuations. (See Scanner::EatLineContinuation.) Otherwise, ' include the new line and any additional spaces as trivia. - If startComment = 0 AndAlso TryPeek(Here, ch) AndAlso Not IsNewLine(ch) Then + If startComment = 0 AndAlso Peep(Here, ch) AndAlso Not IsNewLine(ch) Then tList.Add(MakeEndOfLineTrivia(GetText(newLine))) If spaces > 0 Then @@ -588,13 +587,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Friend Function ScanMultilineTrivia() As SyntaxList(Of VisualBasicSyntaxNode) Dim ch As Char - If Not TryPeek(ch) Then + If Not Peep(ch) Then Return Nothing End If ' optimization for a common case ' the ASCII range between ': and ~ , with exception of except "'", "_" and R cannot start trivia - If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsNoneOf("'"c, "_"c, "R"c, "r"c) Then + If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsntFrom("'"c, "_"c, "R"c, "r"c) Then Return Nothing End If @@ -612,7 +611,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' Private Function TryScanSinglePieceOfMultilineTrivia(tList As SyntaxListBuilder) As Boolean Dim ch As Char - If TryPeek(ch) Then + If Peep(ch) Then Dim atNewLine = IsAtNewLine() @@ -672,7 +671,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' check for # Private Function StartsDirective(Here As Integer) As Boolean Dim ch As Char - Return TryPeek(Here, ch) AndAlso IsHash(ch) + Return Peep(Here, ch) AndAlso IsHash(ch) End Function Private Function IsAtNewLine() As Boolean @@ -713,7 +712,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Sub ScanSingleLineTriviaInXmlDoc(tList As SyntaxListBuilder) Dim c As Char - If TryPeek(c) Then + If Peep(c) Then Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -746,7 +745,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Sub ScanWhitespaceAndLineContinuations(tList As SyntaxListBuilder) Dim c As Char - If TryPeek(c) AndAlso IsWhitespace(c) Then + If Peep(c) AndAlso IsWhitespace(c) Then tList.Add(ScanWhitespace(1)) ' collect { lineCont, ws } While ScanLineContinuation(tList) @@ -816,7 +815,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' Case 3 is required to parse single line if's and numeric labels. ' Case 4 is required to limit explicit line continuations to single new line Dim ch As Char - If TryPeek(ch) Then + If Peep(ch) Then Dim startOfTerminatorTrivia = _lineBufferOffset @@ -829,7 +828,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' collect { ws, colon } Do Dim len = GetWhitespaceLength(0) - If Not TryPeek(len, ch) Then + If Not Peep(len, ch) Then Exit Do End If @@ -869,7 +868,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetWhitespaceLength(len As Integer) As Integer ' eat until linebreak or nonwhitespace Dim c As Char - While TryPeek(len, c) AndAlso IsWhitespace(c) + While Peep(len, c) AndAlso IsWhitespace(c) len += 1 End While Return len @@ -878,7 +877,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function GetXmlWhitespaceLength(len As Integer) As Integer ' eat until linebreak or nonwhitespace Dim c As Char - While TryPeek(len, c) AndAlso IsXmlWhitespace(c) + While Peep(len, c) AndAlso IsXmlWhitespace(c) len += 1 End While Return len @@ -903,31 +902,30 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Sub EatWhitespace() Dim c As Char #If DEBUG Then - Dim ok = TryPeek(c) + Dim ok = Peep(c) Debug.Assert(ok) Debug.Assert(IsWhitespace(c)) #End If AdvanceChar() ' eat until linebreak or nonwhitespace - While TryPeek(c) AndAlso IsWhitespace(c) + While Peep(c) AndAlso IsWhitespace(c) AdvanceChar() End While End Sub Private Function PeekStartComment(i As Integer) As Integer Dim ch As Char - If TryPeek(i, ch) Then - Dim c2, c3 As Char + If Peep(i, ch) Then If IsSingleQuote(ch) Then Return 1 - ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso - TryPeek(i + 2, c2) AndAlso MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso - MatchOneOrAnotherOrFullwidth(c2, "M"c, "m"c) Then + ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso Peep(i + 2, ch) AndAlso + MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso + MatchOneOrAnotherOrFullwidth(ch, "M"c, "m"c) Then - If Not TryPeek(i + 3, c3) OrElse IsNewLine(c3) Then + If Not Peep(i + 3, ch) OrElse IsNewLine(ch) Then ' have only 'REM' Return 3 - ElseIf Not IsIdentifierPartCharacter(c3) Then + ElseIf Not IsIdentifierPartCharacter(ch) Then ' have 'REM ' Return 4 End If @@ -946,7 +944,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' eat all chars until EoL Dim c As Char - While TryPeek(length, c) AndAlso Not IsNewLine(c) + While Peep(length, c) AndAlso Not IsNewLine(c) length += 1 End While @@ -1010,7 +1008,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim dl = ScanDateLiteral(precedingTrivia) Return If(dl, MakeHashToken(precedingTrivia, fullWidth)) Case "&"c - If TryPeek(1, c) AndAlso BeginsBaseLiteral(c) Then + If Peep(1, c) AndAlso BeginsBaseLiteral(c) Then Return ScanNumericLiteral(precedingTrivia) End If If TrySkipFollowingEquals(lengthWithMaybeEquals) Then @@ -1110,7 +1108,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c Return ScanIdentifierOrKeyword(precedingTrivia) Case "_"c - If TryPeek(1, c) AndAlso IsIdentifierPartCharacter(c) Then + If Peep(1, c) AndAlso IsIdentifierPartCharacter(c) Then Return ScanIdentifierOrKeyword(precedingTrivia) End If Dim err As ERRID = ERRID.ERR_ExpectedIdentifier @@ -1129,7 +1127,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc) End If Case "$"c, FULLWIDTH_DOLLAR_SIGN - If Not fullWidth AndAlso TryPeek(1, c) AndAlso IsDoubleQuote(c) Then + If Not fullWidth AndAlso Peep(1, c) AndAlso IsDoubleQuote(c) Then Return MakePunctuationToken(precedingTrivia, 2, SyntaxKind.DollarSignDoubleQuoteToken) End If End Select @@ -1156,7 +1154,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' the beginning of a token Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim ch As Char - If Not TryPeek(ch) Then + If Not Peep(ch) Then Return MakeEofToken(precedingTrivia) End If Return ScanTokenCommon(precedingTrivia, ch, False) @@ -1174,10 +1172,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim Here = Index Dim eq As Char - While TryPeek(Here, eq) + While Peep(Here, eq) Here += 1 If Not IsWhitespace(eq) Then - If eq.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then + If eq.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then Index = Here Return True Else @@ -1191,9 +1189,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanRightAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean) As SyntaxToken Dim c As Char #If DEBUG Then - Dim ok = TryPeek(c) + Dim ok = Peep(c) Debug.Assert(ok) ' > - Debug.Assert(c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN)) + Debug.Assert(c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN)) #End If Dim length As Integer = 1 @@ -1201,11 +1199,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - If TryPeek(length, c) Then - If c.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then + If Peep(length, c) Then + If c.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 Return MakeGreaterThanEqualsToken(precedingTrivia, length) - ElseIf c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then + ElseIf c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then length += 1 If TrySkipFollowingEquals(length) Then Return MakeGreaterThanGreaterThanEqualsToken(precedingTrivia, length) @@ -1220,25 +1218,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanLeftAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean, scanTrailingTrivia As ScanTriviaFunc) As SyntaxToken Dim c As Char #If DEBUG Then - Dim ok = TryPeek(c) + Dim ok = Peep(c) Debug.Assert(ok) ' < - Debug.Assert(c.IsAnyOf("<"c, FULLWIDTH_LESS_THAN_SIGN)) + Debug.Assert(c.IsFrom("<"c, FULLWIDTH_LESS_THAN_SIGN)) #End If Dim length As Integer = 1 - ' Check for XML tokens - If Not charIsFullWidth AndAlso TryPeek(length, c) Then - + If Not charIsFullWidth AndAlso Peep(length, c) Then Select Case c Case "!"c - Dim c2 As Char - If TryPeek(length + 2, c2) Then + If Peep(length + 2, c) Then Select Case (Peek(length + 1)) Case "-"c - If CanGet(length + 3) AndAlso c2 = "-"c Then + If CanGet(length + 3) AndAlso c = "-"c Then Return XmlMakeBeginCommentToken(precedingTrivia, scanTrailingTrivia) End If - Case "["c If NextAre(length + 2, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, scanTrailingTrivia) @@ -1247,7 +1241,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Case "?"c Return XmlMakeBeginProcessingInstructionToken(precedingTrivia, scanTrailingTrivia) - Case "/"c Return XmlMakeBeginEndElementToken(precedingTrivia, _scanSingleLineTriviaFunc) End Select @@ -1255,18 +1248,18 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // Allow whitespace between the characters of a two-character token. length = GetWhitespaceLength(length) - If TryPeek(length, c) Then - If c.IsAnyOf("="c, FULLWIDTH_EQUALS_SIGN) Then + If Peep(length, c) Then + If c.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then length += 1 Return MakeLessThanEqualsToken(precedingTrivia, length) - ElseIf c.IsAnyOf(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then + ElseIf c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then length += 1 Return MakeLessThanGreaterThanToken(precedingTrivia, length) - ElseIf c.IsAnyOf("<"c, FULLWIDTH_LESS_THAN_SIGN) Then + ElseIf c.IsFrom("<"c, FULLWIDTH_LESS_THAN_SIGN) Then length += 1 - If TryPeek(length, c) Then + If Peep(length, c) Then 'if the second "<" is a part of "<%" - like in "<<%" , we do not want to use it. - If c.IsNoneOf("%"c, FULLWIDTH_PERCENT_SIGN) Then + If c.IsntFrom("%"c, FULLWIDTH_PERCENT_SIGN) Then If TrySkipFollowingEquals(length) Then Return MakeLessThanLessThanEqualsToken(precedingTrivia, length) Else @@ -1310,15 +1303,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function ScanIdentifierOrKeyword(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken - Dim ch As Char - Dim ok = TryPeek(ch) ' Non DEBUG usage see:= If IsConnectorPunctuation(ch) + Dim ch, cx As Char + Dim ok = Peep(ch) ' Non DEBUG usage see:= If IsConnectorPunctuation(ch) Debug.Assert(ok) Debug.Assert(IsIdentifierStartCharacter(ch)) Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller - Dim ch1 As Char - If TryPeek(1, ch1) Then - If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(ch1) Then + If Peep(1, cx) Then + If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(cx) Then Return MakeBadToken(precedingTrivia, 1, ERRID.ERR_ExpectedIdentifier) End If End If @@ -1328,7 +1320,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' // The C++ compiler refuses to inline IsIdentifierCharacter, so the ' // < 128 test is inline here. (This loop gets a *lot* of traffic.) ' TODO: make sure we get good perf here - While TryPeek(len, ch) + While Peep(len, ch) Dim code = Convert.ToUInt16(ch) If code < 128 AndAlso IsNarrowIdentifierCharacter(code) OrElse @@ -1342,16 +1334,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax 'Check for a type character Dim TypeCharacter As TypeCharacter = TypeCharacter.None - If TryPeek(len, ch) Then + If Peep(len, ch) Then FullWidthRepeat: Select Case ch Case "!"c - Dim NextChar As Char ' // If the ! is followed by an identifier it is a dictionary lookup operator, not a type character. - If TryPeek(len + 1, NextChar) Then - If IsIdentifierStartCharacter(NextChar) OrElse - MatchOneOrAnotherOrFullwidth(NextChar, "["c, "]"c) Then + If Peep(len + 1, cx) Then + If IsIdentifierStartCharacter(cx) OrElse + MatchOneOrAnotherOrFullwidth(cx, "["c, "]"c) Then Exit Select End If End If @@ -1426,9 +1417,9 @@ FullWidthRepeat: Private Function ScanBracketedIdentifier(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim ch As Char #If DEBUG Then - Dim ok = TryPeek(ch) + Dim ok = Peep(ch) Debug.Assert(ok) ' [ - Debug.Assert(ch.IsAnyOf("["c, FULLWIDTH_LEFT_SQUARE_BRACKET)) + Debug.Assert(ch.IsFrom("["c, FULLWIDTH_LEFT_SQUARE_BRACKET)) #End If Dim IdStart As Integer = 1 @@ -1436,25 +1427,24 @@ FullWidthRepeat: Dim InvalidIdentifier As Boolean = False - If Not TryPeek(Here, ch) Then + If Not Peep(Here, ch) Then Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_MissingEndBrack) End If - Dim c1 As Char + Dim cx As Char ' check if we can start an ident. If Not IsIdentifierStartCharacter(ch) OrElse (IsConnectorPunctuation(ch) AndAlso - Not (TryPeek(Here + 1, c1) AndAlso IsIdentifierPartCharacter(c1))) Then + Not (Peep(Here + 1, cx) AndAlso IsIdentifierPartCharacter(cx))) Then InvalidIdentifier = True End If ' check ident until ] - Dim [Next] As Char - While TryPeek(Here, [Next]) + While Peep(Here, cx) - If [Next].IsAnyOf("]"c, FULLWIDTH_RIGHT_SQUARE_BRACKET) Then + If cx.IsFrom("]"c, FULLWIDTH_RIGHT_SQUARE_BRACKET) Then Dim IdStringLength As Integer = Here - IdStart If IdStringLength > 0 AndAlso Not InvalidIdentifier Then @@ -1476,9 +1466,9 @@ FullWidthRepeat: ' // The sequence "[]" does not define a valid identifier. Return MakeBadToken(precedingTrivia, Here + 1, ERRID.ERR_ExpectedIdentifier) End If - ElseIf IsNewLine([Next]) Then + ElseIf IsNewLine(cx) Then Exit While - ElseIf Not IsIdentifierPartCharacter([Next]) Then + ElseIf Not IsIdentifierPartCharacter(cx) Then InvalidIdentifier = True Exit While End If @@ -1501,7 +1491,7 @@ FullWidthRepeat: Private Function ScanNumericLiteral(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken Dim ch As Char - Dim ok = TryPeek(ch) + Dim ok = Peep(ch) Debug.Assert(ok) Dim Here As Integer = 0 @@ -1516,9 +1506,9 @@ FullWidthRepeat: ' // First read a leading base specifier, if present, followed by a sequence of zero ' // or more digits. - If ch.IsAnyOf("&"c, FULLWIDTH_AMPERSAND) Then + If ch.IsFrom("&"c, FULLWIDTH_AMPERSAND) Then Here += 1 - If Not TryPeek(Here, ch) Then ch = ChrW(0) + If Not Peep(Here, ch) Then ch = ChrW(0) FullWidthRepeat: Select Case ch @@ -1527,7 +1517,7 @@ FullWidthRepeat: IntegerLiteralStart = Here Base = LiteralBase.Hexadecimal - While TryPeek(Here, ch) AndAlso IsHexDigit(ch) + While Peep(Here, ch) AndAlso IsHexDigit(ch) Here += 1 End While @@ -1536,7 +1526,7 @@ FullWidthRepeat: IntegerLiteralStart = Here Base = LiteralBase.Octal - While TryPeek(Here, ch) AndAlso IsOctalDigit(ch) + While Peep(Here, ch) AndAlso IsOctalDigit(ch) Here += 1 End While @@ -1551,7 +1541,7 @@ FullWidthRepeat: Else ' no base specifier - just go through decimal digits. IntegerLiteralStart = Here - While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While End If @@ -1561,16 +1551,16 @@ FullWidthRepeat: ' // Unless there was an explicit base specifier (which indicates an integer literal), ' // read the rest of a float literal. - If Base = LiteralBase.Decimal AndAlso TryPeek(Here, ch) Then + If Base = LiteralBase.Decimal AndAlso Peep(Here, ch) Then ' // First read a '.' followed by a sequence of one or more digits. Dim cx As Char - If ch.IsAnyOf("."c, FULLWIDTH_FULL_STOP) AndAlso - (TryPeek(Here + 1, cx) AndAlso IsDecimalDigit(cx)) Then + If ch.IsFrom("."c, FULLWIDTH_FULL_STOP) AndAlso + (Peep(Here + 1, cx) AndAlso IsDecimalDigit(cx)) Then Here += 2 ' skip dot and first digit ' all following decimal digits belong to the literal (fractional part) - While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While literalKind = NumericLiteralKind.Float @@ -1578,16 +1568,16 @@ FullWidthRepeat: ' // Read an exponent symbol followed by an optional sign and a sequence of ' // one or more digits. - If TryPeek(Here, cx) AndAlso BeginsExponent(cx) Then + If Peep(Here, cx) AndAlso BeginsExponent(cx) Then Here += 1 - If TryPeek(Here, ch) AndAlso MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then + If Peep(Here, ch) AndAlso MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then Here += 1 End If - If TryPeek(Here, cx) AndAlso IsDecimalDigit(cx) Then + If Peep(Here, cx) AndAlso IsDecimalDigit(cx) Then Here += 1 - While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Here += 1 End While Else @@ -1606,7 +1596,7 @@ FullWidthRepeat: Dim TypeCharacter As TypeCharacter = TypeCharacter.None - If TryPeek(Here, ch) Then + If Peep(Here, ch) Then FullWidthRepeat2: Select Case ch @@ -1682,7 +1672,7 @@ FullWidthRepeat2: literalKind = NumericLiteralKind.Decimal ' check if this was not attempt to use obsolete exponent - If TryPeek(Here + 1, ch) Then + If Peep(Here + 1, ch) Then If IsDecimalDigit(ch) OrElse MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_ObsoleteExponent) End If @@ -1693,7 +1683,7 @@ FullWidthRepeat2: Case "U"c, "u"c Dim NextChar As Char - If literalKind <> NumericLiteralKind.Float AndAlso TryPeek(Here + 1, NextChar) Then + If literalKind <> NumericLiteralKind.Float AndAlso Peep(Here + 1, NextChar) Then 'unsigned suffixes - US, UL, UI If MatchOneOrAnotherOrFullwidth(NextChar, "S"c, "s"c) Then TypeCharacter = TypeCharacter.UShortLiteral @@ -1867,14 +1857,14 @@ FullWidthRepeat2: Private Function ScanIntLiteral(ByRef ReturnValue As Integer, ByRef Here As Integer) As Boolean Debug.Assert(Here >= 0) Dim ch As Char - If Not TryPeek(Here, ch) OrElse Not IsDecimalDigit(ch) Then + If Not Peep(Here, ch) OrElse Not IsDecimalDigit(ch) Then Return False End If Dim IntegralValue As Integer = IntegralLiteralCharacterValue(ch) Here += 1 - While TryPeek(Here, ch) AndAlso IsDecimalDigit(ch) + While Peep(Here, ch) AndAlso IsDecimalDigit(ch) Dim nextDigit = IntegralLiteralCharacterValue(ch) If IntegralValue < 214748364 OrElse (IntegralValue = 214748364 AndAlso nextDigit < 8) Then @@ -1924,7 +1914,7 @@ FullWidthRepeat2: ' // If we see a /, then it's a date Dim c As Char - If TryPeek(Here, c) AndAlso IsDateSeparatorCharacter(c) Then + If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then Dim FirstDateSeparator As Integer = Here ' // We've got a date @@ -1944,11 +1934,10 @@ FullWidthRepeat2: End If ' Do we have a day value? - Dim c1 As Char - If TryPeek(Here, c1) AndAlso IsDateSeparatorCharacter(c1) Then + If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then ' // Check to see they used a consistent separator - If c1 <> Peek(FirstDateSeparator) Then + If c <> Peek(FirstDateSeparator) Then GoTo baddate End If @@ -1970,11 +1959,10 @@ FullWidthRepeat2: End If ' // Do we have a year value? - Dim c1 As Char - If TryPeek(Here, c1) AndAlso IsDateSeparatorCharacter(c1) Then + If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then ' // Check to see they used a consistent separator - If c1 <> Peek(FirstDateSeparator) Then + If c <> Peek(FirstDateSeparator) Then GoTo baddate End If @@ -2013,8 +2001,7 @@ FullWidthRepeat2: If HaveTimeValue Then ' // Do we see a :? - Dim c1 As Char - If TryPeek(Here, c1) AndAlso IsColon(c1) Then + If Peep(Here, c) AndAlso IsColon(c) Then Here += 1 ' // Now let's get the minute value @@ -2026,8 +2013,7 @@ FullWidthRepeat2: HaveMinuteValue = True ' // Do we have a second value? - Dim c2 As Char - If TryPeek(Here, c2) AndAlso IsColon(c2) Then + If Peep(Here, c) AndAlso IsColon(c) Then ' // Yes. HaveSecondValue = True Here += 1 @@ -2042,21 +2028,20 @@ FullWidthRepeat2: ' // Check AM/PM - If TryPeek(Here, c1) Then - If c1.IsAnyOf("A"c, FULLWIDTH_LATIN_CAPITAL_LETTER_A, + If Peep(Here, c) Then + If c.IsFrom("A"c, FULLWIDTH_LATIN_CAPITAL_LETTER_A, "a"c, FULLWIDTH_LATIN_SMALL_LETTER_A) Then HaveAM = True Here += 1 - ElseIf c1.IsAnyOf("P"c, FULLWIDTH_LATIN_CAPITAL_LETTER_P, + ElseIf c.IsFrom("P"c, FULLWIDTH_LATIN_CAPITAL_LETTER_P, "p"c, FULLWIDTH_LATIN_SMALL_LETTER_P) Then HavePM = True Here += 1 End If - Dim c2 As Char - If TryPeek(Here, c2) AndAlso (HaveAM OrElse HavePM) Then - If c2.IsAnyOf("M"c, FULLWIDTH_LATIN_CAPITAL_LETTER_M, + If Peep(Here, c) AndAlso (HaveAM OrElse HavePM) Then + If c.IsFrom("M"c, FULLWIDTH_LATIN_CAPITAL_LETTER_M, "m"c, FULLWIDTH_LATIN_SMALL_LETTER_M) Then Here = GetWhitespaceLength(Here + 1) @@ -2074,7 +2059,7 @@ FullWidthRepeat2: End If End If - If Not TryPeek(Here, c) OrElse Not IsHash(c) Then + If Not Peep(Here, c) OrElse Not IsHash(c) Then GoTo baddate End If @@ -2179,12 +2164,11 @@ FullWidthRepeat2: baddate: ' // If we can find a closing #, then assume it's a malformed date, ' // otherwise, it's not a date - Dim ch As Char - While TryPeek(Here, ch) AndAlso Not (IsHash(ch) OrElse IsNewLine(ch)) + While Peep(Here, c) AndAlso Not (IsHash(c) OrElse IsNewLine(c)) Here += 1 End While - If Not TryPeek(Here, ch) OrElse IsNewLine(ch) Then + If Not Peep(Here, c) OrElse IsNewLine(c) Then ' // No closing # Return Nothing Else @@ -2199,15 +2183,15 @@ baddate: Debug.Assert(IsDoubleQuote(Peek)) Dim length As Integer = 1 - Dim ch, cx As Char + Dim ch As Char Dim followingTrivia As SyntaxList(Of VisualBasicSyntaxNode) ' // Check for a Char literal, which can be of the form: ' // """"c or ""c - If TryPeek(3, ch) AndAlso IsDoubleQuote(Peek(2)) Then + If Peep(3, ch) AndAlso IsDoubleQuote(Peek(2)) Then If IsDoubleQuote(Peek(1)) Then - If IsDoubleQuote(ch) AndAlso TryPeek(4, cx) AndAlso IsLetterC(cx) Then + If IsDoubleQuote(ch) AndAlso Peep(4, ch) AndAlso IsLetterC(ch) Then ' // Double-quote Char literal: """"c Return MakeCharacterLiteralToken(precedingTrivia, """"c, 5) End If @@ -2218,16 +2202,16 @@ baddate: End If End If - If TryPeek(2, ch) AndAlso IsDoubleQuote(Peek(1)) AndAlso IsLetterC(ch) Then + If Peep(2, ch) AndAlso IsDoubleQuote(Peek(1)) AndAlso IsLetterC(ch) Then ' // Error. ""c is not a legal char constant Return MakeBadToken(precedingTrivia, 3, ERRID.ERR_IllegalCharConstant) End If Dim scratch = GetScratch() - While TryPeek(length, ch) + While Peep(length, ch) If IsDoubleQuote(ch) Then - If TryPeek(length + 1, ch) Then + If Peep(length + 1, ch) Then If IsDoubleQuote(ch) Then ' // An escaped double quote scratch.Append(""""c) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index fa06b5c9f7dca..74921c99e43aa 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -96,11 +96,22 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return p End Function - Friend Function TryPeek(at As Integer, ByRef ch As Char) As Boolean + ''' + ''' + ''' + ''' How far in front do what to try "peeking" at? + ''' (Passback) the character at that location if possible. + ''' If not the contents are left unaffected. + ''' NOTE: This is different behaviour to the Try methods that always return a value. + ''' + ''' Returns True if it possible to peek at, otherwise returns False. + Friend Function Peep(at As Integer, ByRef ch As Char) As Boolean ' CanGet(at) Debug.Assert(_lineBufferOffset + at >= 0) Debug.Assert(at >= -MaxCharsLookBehind) - If _lineBufferOffset + at >= _bufferLen Then Return False + If _lineBufferOffset + at >= _bufferLen Then + Return False + End If ' Peek(at) at += _lineBufferOffset Dim page = _curPage @@ -111,10 +122,19 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Return True End Function - - Friend Function TryPeek( ByRef ch As Char) As Boolean + ''' + ''' + ''' + ''' (Passback) the character at that location if possible. + ''' If not the contents are left unaffected. + ''' NOTE: This is different behaviour to the Try methods that always return a value. + ''' + ''' Returns True if it possible to peek at, otherwise returns False. + Friend Function Peep(ByRef ch As Char) As Boolean ' CanGet() - If _lineBufferOffset >= _bufferLen Then Return False + If _lineBufferOffset >= _bufferLen Then + Return False + End If ' Peek() Dim page = _curPage Dim position = _lineBufferOffset diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb index b96fe897fb62b..07e521d3c856c 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb @@ -20,24 +20,23 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim offset = leadingTriviaLength Dim length As Integer Dim c As Char - If Not TryPeek(offset, c) Then + If Not Peep(offset, c) Then Return MakeEndOfInterpolatedStringToken() End If ' This should only ever happen for $" or } - Debug.Assert(leadingTriviaLength = 0 OrElse c.IsAnyOf("$"c, FULLWIDTH_DOLLAR_SIGN) OrElse IsRightCurlyBracket(c)) + Debug.Assert(leadingTriviaLength = 0 OrElse c.IsFrom("$"c, FULLWIDTH_DOLLAR_SIGN) OrElse IsRightCurlyBracket(c)) ' Another } may follow the close brace of an interpolation if the interpolation lacked a format clause. ' This is because the normal escaping rules only apply when parsing the format string. Debug.Assert(Not CanGet(1) OrElse Peek(offset + 1) <> c OrElse IsRightCurlyBracket(c), "Escape sequence not detected.") Dim scanTrailingTrivia As Boolean - Dim c1 As Char Select Case c Case "$"c, FULLWIDTH_DOLLAR_SIGN - If TryPeek(offset + 1, c1) AndAlso IsDoubleQuote(c1) Then + If Peep(offset + 1, c) AndAlso IsDoubleQuote(c) Then kind = SyntaxKind.DollarSignDoubleQuoteToken length = 2 scanTrailingTrivia = False ' Trailing whitespace should be scanned as interpolated string text. @@ -111,20 +110,20 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function Private Function IsInterpolatedStringPunctuation(Optional offset As Integer = 0) As Boolean - Dim c, c1 As Char - If Not TryPeek(offset, c) Then Return False + Dim c As Char + If Not Peep(offset, c) Then Return False If IsLeftCurlyBracket(c) Then - Return Not TryPeek(offset + 1, c1) OrElse Not IsLeftCurlyBracket(c1) + Return Not Peep(offset + 1, c) OrElse Not IsLeftCurlyBracket(c) ElseIf IsRightCurlyBracket(c) Then - Return Not TryPeek(offset + 1, c1) OrElse Not IsRightCurlyBracket(c1) + Return Not Peep(offset + 1, c) OrElse Not IsRightCurlyBracket(c) ElseIf IsDoubleQuote(c) 'A subtle difference between this case and the one above. ' In both interpolated and literal strings the two quote characters used in an escape sequence don't have to match. ' It's enough that the next character is *a* quote char. It doesn't have to be the same quote. ' If we want to preserve consistency the quotes need to be special cased. - Return Not TryPeek(offset + 1, c1) OrElse Not IsDoubleQuote(c1) + Return Not Peep(offset + 1, c) OrElse Not IsDoubleQuote(c) Else Return False @@ -137,15 +136,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim offset = 0 Dim pendingWhitespace = 0 Dim valueBuilder = GetScratch() - Dim c, c1 As Char - Do While TryPeek(offset, c) + Dim c As Char + Do While Peep(offset, c) ' Any combination of fullwidth and ASCII curly braces of the same direction is an escaping sequence for the corresponding ASCII curly brace. ' We insert that curly brace doubled and because this is the escaping sequence understood by String.Format, that will be replaced by a single brace. ' This is deliberate design and it aligns with existing rules for double quote escaping in strings. If IsLeftCurlyBracket(c) Then - - If TryPeek(offset + 1, c1) AndAlso IsLeftCurlyBracket(c1) Then + If Peep(offset + 1, c) AndAlso IsLeftCurlyBracket(c) Then ' This is an escape sequence. valueBuilder.Append("{{") @@ -158,8 +156,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do ElseIf IsRightCurlyBracket(c) Then - - If TryPeek(offset + 1, c1) AndAlso IsRightCurlyBracket(c1) Then + If Peep(offset + 1, c) AndAlso IsRightCurlyBracket(c) Then ' This is an escape sequence. valueBuilder.Append("}}") @@ -172,8 +169,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do ElseIf IsDoubleQuote(c) - - If TryPeek(offset + 1, c1) AndAlso IsDoubleQuote(c1) Then + If Peep(offset + 1, c) AndAlso IsDoubleQuote(c) Then ' This is a VB double quote escape. Oddly enough this logic allows mixing and matching of ' smart and dumb double quotes in any order. Regardless we always emit as a standard double quote. ' This is consistent with their handling in string literals. diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb index f4e0a630d0e47..d72bb0e4abd4e 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb @@ -16,15 +16,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanXmlTrivia(c As Char) As SyntaxList(Of VisualBasicSyntaxNode) Debug.Assert(Not IsScanningXmlDoc) - Debug.Assert(c.IsAnyOf( CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION)) + Debug.Assert(c.IsFrom( CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION)) Dim builder = _triviaListPool.Allocate Dim len = 0 Do - If c.IsAnyOf(" "c, CHARACTER_TABULATION) Then + If c.IsFrom(" "c, CHARACTER_TABULATION) Then len += 1 - ElseIf c.IsAnyOf(CARRIAGE_RETURN, LINE_FEED) Then + ElseIf c.IsFrom(CARRIAGE_RETURN, LINE_FEED) Then If len > 0 Then builder.Add(MakeWhiteSpaceTrivia(GetText(len))) len = 0 @@ -34,7 +34,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Exit Do End If - If Not TryPeek(len, c) Then + If Not Peep(len, c) Then Exit Do End If Loop @@ -69,7 +69,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing Dim c As Char - While TryPeek(c) + While Peep(c) Select Case (c) ' // Whitespace @@ -113,11 +113,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Case "<"c Dim ch As Char - If TryPeek(1, ch) Then + If Peep(1, ch) Then Dim c2 As Char Select Case ch Case "!"c - If TryPeek(2,c2) Then + If Peep(2,c2) Then Select Case c2 Case "-"c If NextIs(3,"-"c) Then @@ -207,7 +207,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '// Example ' This is a comment Private Function ScanXmlForPossibleStatement(state As ScannerState) As Boolean Dim c As Char - If Not TryPeek(c) Then + If Not Peep(c) Then Return False End If @@ -238,7 +238,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If name.PossibleKeywordKind <> SyntaxKind.XmlNameToken Then leadingTrivia = ScanSingleLineTrivia() c = Peek() - possibleStatement = c.IsAnyOf("("c, FULLWIDTH_LEFT_PARENTHESIS) + possibleStatement = c.IsFrom("("c, FULLWIDTH_LEFT_PARENTHESIS) End If End If @@ -308,7 +308,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim scratch = GetScratch() Dim c As Char - While TryPeek(Here, c) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -341,22 +341,22 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Debug.Assert(Here = 0) Dim ch As Char - If TryPeek(1, ch) Then + If Peep(1, ch) Then Select Case ch Case "!"c Dim c2 As Char - If TryPeek(2, c2) Then + If Peep(2, c2) Then Select Case c2 Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If NextAre(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If End Select @@ -490,7 +490,7 @@ ScanChars: Dim Here = 0 Dim c As Char - While TryPeek(Here, c) + While Peep(Here, c) Select Case c Case CARRIAGE_RETURN, LINE_FEED @@ -507,7 +507,7 @@ ScanChars: Return XmlMakeCommentToken(precedingTrivia, Here) End If - If TryPeek(Here + 2, c) Then + If Peep(Here + 2, c) Then Here += 2 ' // if > is not found then this is an error. Return the -- string @@ -578,7 +578,7 @@ ScanChars: Dim scratch = GetScratch() Dim Here = 0 Dim c As Char - While TryPeek(Here, c) + While Peep(Here, c) Select Case c Case CARRIAGE_RETURN, LINE_FEED Here = SkipLineBreak(c, Here) @@ -640,7 +640,7 @@ ScanChars: Dim precedingTrivia = _triviaListPool.Allocate(Of VisualBasicSyntaxNode)() Dim result As SyntaxToken Dim c As Char - If state = ScannerState.StartProcessingInstruction AndAlso TryPeek(c) Then + If state = ScannerState.StartProcessingInstruction AndAlso Peep(c) Then ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ Select Case c @@ -651,7 +651,7 @@ ScanChars: End If Dim Here = 0 - While TryPeek(Here, c) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -710,7 +710,7 @@ CleanUp: Dim precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing Dim c As Char - While TryPeek(c) + While Peep(c) Select Case (c) ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ @@ -721,7 +721,7 @@ CleanUp: Case "<"c Dim ch As Char - If TryPeek(1, ch) Then + If Peep(1, ch) Then Select Case ch Case "!"c If NextAre(2, "--") Then @@ -785,7 +785,7 @@ CleanUp: Dim Here = 0 Dim scratch = GetScratch() - While TryPeek(Here,c) + While Peep(Here,c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION If Here > 0 Then @@ -875,7 +875,7 @@ ScanChars: Dim Here = 0 Dim scratch = GetScratch() Dim c As Char - While TryPeek(Here, c) + While Peep(Here, c) If c = terminatingChar Or c = altTerminatingChar Then If Here > 0 Then result = XmlMakeAttributeDataToken(precedingTrivia, Here, scratch) @@ -977,7 +977,7 @@ CleanUp: Debug.Assert(CanGet(Here)) Debug.Assert(Peek(Here) = c1) Dim c2 As Char - If IsHighSurrogate(c1) AndAlso TryPeek(Here + 1, c2) Then + If IsHighSurrogate(c1) AndAlso Peep(Here + 1, c2) Then If IsLowSurrogate(c2) Then Return New XmlCharResult(c1, c2) End If @@ -1017,7 +1017,7 @@ CleanUp: Private Function ScanXmlChar(Here As Integer) As XmlCharResult Debug.Assert(Here >= 0) Dim c As Char - Dim ok =TryPeek(Here, c) + Dim ok =Peep(Here, c) Debug.Assert(ok) If Not isValidUtf16(c) Then @@ -1050,7 +1050,7 @@ CleanUp: 'TODO - Fix ScanXmlNCName to conform to XML spec instead of old loose scanning. Dim c As Char - While TryPeek(Here, c) + While Peep(Here, c) Select Case (c) Case ":"c, " "c, CHARACTER_TABULATION, LINE_FEED, CARRIAGE_RETURN, @@ -1115,7 +1115,7 @@ CreateNCNameToken: Debug.Assert( NextIs(0,"&"c)) Dim c, c1 As Char ' skip 1 char for "&" - If TryPeek(1, c) Then + If Peep(1, c) Then Select Case (c) Case "#"c Dim Here = 2 ' skip "&#" @@ -1141,7 +1141,7 @@ CreateNCNameToken: Case "a"c ' // & ' // ' - If TryPeek(4, c1) AndAlso NextAre(2,"mp") Then + If Peep(4, c1) AndAlso NextAre(2,"mp") Then If c1 = ";"c Then Return XmlMakeAmpLiteralToken(precedingTrivia) Else @@ -1150,7 +1150,7 @@ CreateNCNameToken: Return DirectCast(noSemicolon.SetDiagnostics({noSemicolonError}), XmlTextTokenSyntax) End If - ElseIf TryPeek(5, c1) AndAlso NextAre(2,"pos") Then + ElseIf Peep(5, c1) AndAlso NextAre(2,"pos") Then If c1 = ";"c Then Return XmlMakeAposLiteralToken(precedingTrivia) Else @@ -1162,7 +1162,7 @@ CreateNCNameToken: Case "l"c ' // < - If TryPeek(3, c1) AndAlso NextIs(2,"t"c) Then + If Peep(3, c1) AndAlso NextIs(2,"t"c) Then If c1 = ";"c Then Return XmlMakeLtLiteralToken(precedingTrivia) Else @@ -1174,7 +1174,7 @@ CreateNCNameToken: Case "g"c ' // > - If TryPeek(3, c1) AndAlso NextIs(2,"t"c) Then + If Peep(3, c1) AndAlso NextIs(2,"t"c) Then If c1 = ";"c Then Return XmlMakeGtLiteralToken(precedingTrivia) Else @@ -1186,7 +1186,7 @@ CreateNCNameToken: Case "q"c ' // " - If TryPeek(5, c1) AndAlso NextAre(2,"uot") Then + If Peep(5, c1) AndAlso NextAre(2,"uot") Then If c1 = ";"c Then Return XmlMakeQuotLiteralToken(precedingTrivia) Else @@ -1208,7 +1208,7 @@ CreateNCNameToken: Private Function ScanXmlCharRef(ByRef index As Integer) As XmlCharResult Debug.Assert(index >= 0) Dim ch As Char - If Not TryPeek(index, ch) Then + If Not Peep(index, ch) Then Return Nothing End If @@ -1218,7 +1218,7 @@ CreateNCNameToken: If ch = "x"c Then Here += 1 - While TryPeek(Here, ch) + While Peep(Here, ch) If XmlCharType.IsHexDigit(ch) Then charRefSb.Append(ch) Else @@ -1235,7 +1235,7 @@ CreateNCNameToken: End If Else - While TryPeek(Here, ch) AndAlso XmlCharType.IsDigit(ch) + While Peep(Here, ch) AndAlso XmlCharType.IsDigit(ch) charRefSb.Append(ch) Here += 1 End While diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index 5a0024bdcab22..37d43c57a4076 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -51,7 +51,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Debug.Assert(IsAtNewLine) ' leading whitespace until we see ''' should be regular whitespace Dim c As Char - If TryPeek(c) AndAlso IsWhitespace(c) Then + If Peep(c) AndAlso IsWhitespace(c) Then Dim ws = ScanWhitespace() tList.Add(ws) End If @@ -149,7 +149,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function TrySkipXmlDocMarker(ByRef len As Integer) As Boolean Dim c As Char Dim Here = len - While TryPeek(Here, c) AndAlso IsWhitespace(c) + While Peep(Here, c) AndAlso IsWhitespace(c) Here += 1 End While @@ -182,7 +182,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim len = 0 Do - If c.IsAnyOf(" "c, CHARACTER_TABULATION) Then + If c.IsFrom(" "c, CHARACTER_TABULATION) Then len += 1 ElseIf IsNewLine(c) Then @@ -237,7 +237,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim Here As Integer = 0 Dim scratch = GetScratch() Dim c As Char - While TryPeek(Here, c) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -277,23 +277,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Debug.Assert(Here = 0) - Dim ch As Char - If TryPeek(1, ch) Then - Select Case ch + If Peep(1, c) Then + Select Case c Case "!"c - Dim c2 As Char - If TryPeek(2, c2) Then - Select Case c2 + If Peep(2, c) Then + Select Case c Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If NextAre(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If End Select @@ -366,7 +364,7 @@ ScanChars: End If Dim c As Char - If state = ScannerState.StartProcessingInstruction AndAlso TryPeek(c) Then + If state = ScannerState.StartProcessingInstruction AndAlso Peep(c) Then ' // Whitespace ' // S ::= (#x20 | #x9 | #xD | #xA)+ Select Case c @@ -382,7 +380,7 @@ ScanChars: End If Dim Here = 0 - While TryPeek(Here, c) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED @@ -455,7 +453,7 @@ CleanUp: End If Dim c As Char - While TryPeek(c) + While Peep(c) If Not precedingTrivia.Any AndAlso IsAtNewLine() AndAlso Not Me._doNotRequireXmlDocCommentPrefix Then ' this would indicate that we looked at Trivia, but did not find ' XmlDoc prefix (or we would not be at the line start) @@ -498,22 +496,21 @@ CleanUp: Return XmlMakeDoubleQuoteToken(precedingTrivia, c, isOpening:=True) Case "<"c - Dim ch, c2 As Char - If TryPeek(1, ch) Then - Select Case ch + If Peep(1, c) Then + Select Case c Case "!"c - If TryPeek(2, c2) Then - Select Case c2 + If Peep(2, c) Then + Select Case c Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If NextAre(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(precedingTrivia) End If End Select @@ -542,11 +539,7 @@ CleanUp: Case ")"c Return XmlMakeRightParenToken(precedingTrivia) - Case "!"c, - ";"c, - "#"c, - ","c, - "}"c + Case "!"c, ";"c, "#"c, ","c, "}"c Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) Case ":"c diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb index c98efa4c657e3..fc5767cffcf6d 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb @@ -318,7 +318,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function XmlMakeEndEmbeddedToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax Dim c = Peek - Debug.Assert(c.IsAnyOf("%"c, FULLWIDTH_PERCENT_SIGN)) + Debug.Assert(c.IsFrom("%"c, FULLWIDTH_PERCENT_SIGN)) Debug.Assert(Peek(1) = ">"c) Dim spelling As String From cc616edc7a0c8c9de82430ef91915f2d72445f6d Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Tue, 7 Apr 2015 02:17:53 +0100 Subject: [PATCH 6/9] Add attribute to methods. --- .../VisualBasic/Portable/Scanner/Blender.vb | 12 +- .../Portable/Scanner/CharacterInfo.vb | 5 +- .../Portable/Scanner/KeywordTable.vb | 4 +- .../Portable/Scanner/QuickTokenAccumulator.vb | 14 +- .../VisualBasic/Portable/Scanner/Scanner.vb | 11 +- .../Portable/Scanner/ScannerBuffer.vb | 9 +- .../Scanner/ScannerInterpolatedString.vb | 2 +- .../Portable/Scanner/ScannerXml.vb | 316 +++++++++--------- .../Portable/Scanner/TokenStream.vb | 26 +- .../Portable/Scanner/XmlDocComments.vb | 1 + 10 files changed, 206 insertions(+), 194 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb b/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb index e01a47a99bf26..dc241af8f0957 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary Option Strict On @@ -139,7 +139,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return node.GetTrailingTrivia().Any(SyntaxKind.EndOfLineTrivia) Case SyntaxKind.SingleLineIfStatement, SyntaxKind.SingleLineElseClause - ' Steer clear of single-line if's because they they have custom handling of statement + ' Steer clear of single-line if's because they they have custom handling of statement ' terminators that may make it difficult to reuse sub-statements. Return False Case Else @@ -301,7 +301,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Shared Function ShouldCrumble(node As VisualBasicSyntaxNode) As Boolean If TypeOf node Is StructuredTriviaSyntax Then ' Do not crumble into structured trivia content. - ' we will not use any of the parts anyways and + ' we will not use any of the parts anyways and ' evaluation of directives may go out of sync. Return False End If @@ -516,14 +516,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax _currentToken = _currentToken.With(nextPreprocessorState) End If - ' this will discard any prefetched tokens, including current. + ' this will discard any prefetched tokens, including current. ' We do not need them since we moved to completely new node. MyBase.MoveToNextSyntaxNode() TryPopNode() ' at this point, all three pointers (position in old tree, position in text, position in parser) - ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState) + ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState) ' should point to same position (in sync) End Sub @@ -537,7 +537,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' just move forward _lineBufferOffset = _lineBufferOffset + _curNodeLength - ' this will just verify that we do not have any prefetched tokens, including current. + ' this will just verify that we do not have any prefetched tokens, including current. ' otherwise advancing linebuffer offeset could go out of sync with token stream. MyBase.MoveToNextSyntaxNodeInTrivia() diff --git a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb index 575ca17d7839a..1f6f7bc8b621e 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Strict On Option Explicit On @@ -339,7 +339,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic End Function ''' - ''' Creates a half width form Unicode character string. + ''' Creates a half width form Unicode character string. ''' ''' The text representing the original identifier. This can be in full width or half width Unicode form. ''' A string representing the text in a half width Unicode form. @@ -519,6 +519,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) AndAlso (c <> c3) End Function + End Module End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb index c45ede3b05bb2..bd05eb43d7068 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb @@ -269,7 +269,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax Public kdIsQueryClause As Boolean '// Note: "CanFollowExpr" says whether this token can come after an expression. - '// e.g. "Dim x = From i In Select i" is valid, therefore "Select" can come follow an expression. + '// e.g. "Dim x = From i In Select i" is valid, therefore "Select" can come follow an expression. '// The complete list was discovered through the tool in vb\Language\Tools\VBGrammarAnalyzer\vbgrammar.html '// If you add keywords, then make sure that they're added to the official language grammar, and re-run the tool. Public kdCanFollowExpr As Boolean @@ -305,7 +305,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax Private Shared Function EnsureHalfWidth(s As String) As String Dim result As Char() = Nothing For i As Integer = 0 To s.Length - 1 - Dim ch As Char = s(i) + Dim ch As Char = s(i) If SyntaxFacts.IsFullWidth(ch) Then ch = SyntaxFacts.MakeHalfWidth(ch) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb b/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb index 298c37f2058d5..32a0f8819a232 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb @@ -16,16 +16,16 @@ Imports Microsoft.CodeAnalysis.VisualBasic.Syntax Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax '' The QuickTokenAccumulator is a small mini-tokenizer that may fail. It consumes characters and '' eventually either decides that either it found a complete token (including the trivia on either - '' side), or it gives up and says that the full scanner should take an attempt. It also accumulates the + '' side), or it gives up and says that the full scanner should take an attempt. It also accumulates the '' token into a character buffer and computes a hash code. The entire tokenization is done by a single '' routine without any memory allocations to keep it very fast. - '' + '' '' Currently it only handles two cases: '' optional-whitespace keyword-or-identifier optional-whitespace '' optional-whitespace single-char-punctuation optional-whitespace - '' + '' '' where the whitespace does not include newlines. - ' + '' '' The VB tokenization rules are complex, and care needs to be taken in constructing the quick '' tokenization. For example "REM" begins a comment, so it can't be tokenized as a keyword or identifier. '' Similar problems arise with multi-character punctuation tokens, which can have embedded spaces. @@ -63,7 +63,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax 'TODO: why : and ; are complex? (8th row, 3 and 4) - ' The following table classifies the first &H180 Unicode characters. + ' The following table classifies the first &H180 Unicode characters. ' R and r are marked as COMPLEX so that quick-scanning doesn't stop after "REM". ' # is marked complex as it may start directives. ' PERF: Use UShort instead of CharFlags so the compiler can use array literal initialization. @@ -163,7 +163,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' Attempt to scan a single token. ' If it succeeds, return True, and the characters, length, and hashcode of the token - ' can be retrieved by other functions. + ' can be retrieved by other functions. ' If it fails (the token is too complex), return False. Public Function QuickScanToken(allowLeadingMultilineTrivia As Boolean) As QuickScanResult Dim state As AccumulatorState = If(allowLeadingMultilineTrivia, AccumulatorState.InitialAllowLeadingMultilineTrivia, AccumulatorState.Initial) @@ -330,7 +330,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax index += 1 - 'FNV-like hash should work here + 'FNV-like hash should work here 'since these strings are short and mostly ASCII hashCode = (hashCode Xor unicodeValue) * Hash.FnvPrime End While diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index 08f3e6540bd70..9a82b06d6766a 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -233,8 +233,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If _badTokenCount < BadTokenCountLimit Then ' // Don't break up surrogate pairs - Dim c = Peek() - Dim length = If(IsHighSurrogate(c) AndAlso CanGet(1) AndAlso IsLowSurrogate(Peek(1)), 2, 1) + c = Peek() + Dim length = If(IsHighSurrogate(c) AndAlso Peep(1, c) AndAlso IsLowSurrogate(c), 2, 1) token = MakeBadToken(leadingTrivia, length, ERRID.ERR_IllegalChar) Else ' If we get too many characters that we cannot make sense of, absorb the rest of the input. @@ -397,10 +397,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #Region "Buffer helpers" + Private Function NextAre(chars As String) As Boolean Return NextAre(0, chars) End Function + Private Function NextAre(offset As Integer, chars As String) As Boolean Debug.Assert(Not String.IsNullOrEmpty(chars)) Dim n = chars.Length @@ -411,14 +413,17 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return True End Function + Private Function NextIs(offset As Integer, c As Char) As Boolean Return CanGet(offset) AndAlso (Peek(offset) = c) End Function + Private Function CanGet() As Boolean Return _lineBufferOffset < _bufferLen End Function + Private Function CanGet(num As Integer) As Boolean Debug.Assert(_lineBufferOffset + num >= 0) Debug.Assert(num >= -MaxCharsLookBehind) @@ -566,7 +571,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' include the new line and any additional spaces as trivia. If startComment = 0 AndAlso Peep(Here, ch) AndAlso Not IsNewLine(ch) Then - tList.Add(MakeEndOfLineTrivia(GetText(newLine))) If spaces > 0 Then tList.Add(MakeWhiteSpaceTrivia(GetText(spaces))) @@ -1149,7 +1153,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return Nothing End Function - ' at this point it is very likely that we are located at ' the beginning of a token Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index 74921c99e43aa..9d63044c5bca4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -97,7 +97,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End Function ''' - ''' + ''' Peep at the index position, if it is reachable pass the char back ''' ''' How far in front do what to try "peeking" at? ''' (Passback) the character at that location if possible. @@ -105,6 +105,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' NOTE: This is different behaviour to the Try methods that always return a value. ''' ''' Returns True if it possible to peek at, otherwise returns False. + Friend Function Peep(at As Integer, ByRef ch As Char) As Boolean ' CanGet(at) Debug.Assert(_lineBufferOffset + at >= 0) @@ -122,14 +123,16 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Return True End Function + ''' - ''' + ''' Peep at the next index position, if it is reachable pass the char back ''' ''' (Passback) the character at that location if possible. ''' If not the contents are left unaffected. ''' NOTE: This is different behaviour to the Try methods that always return a value. ''' ''' Returns True if it possible to peek at, otherwise returns False. + Friend Function Peep(ByRef ch As Char) As Boolean ' CanGet() If _lineBufferOffset >= _bufferLen Then @@ -265,5 +268,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax End If Return result End Function + End Class + End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb index 07e521d3c856c..075c1e4ad674e 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Imports Microsoft.CodeAnalysis.VisualBasic.SyntaxFacts diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb index d72bb0e4abd4e..a00782ff946f3 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb @@ -16,7 +16,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Private Function ScanXmlTrivia(c As Char) As SyntaxList(Of VisualBasicSyntaxNode) Debug.Assert(Not IsScanningXmlDoc) - Debug.Assert(c.IsFrom( CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION)) + Debug.Assert(c.IsFrom(CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION)) Dim builder = _triviaListPool.Allocate @@ -92,7 +92,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax leadingTrivia = ScanXmlTrivia(c) Case "/"c - If NextIs(1,">"c) Then + If NextIs(1, ">"c) Then Return XmlMakeEndEmptyElementToken(leadingTrivia) End If Return XmlMakeDivToken(leadingTrivia) @@ -117,25 +117,25 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Dim c2 As Char Select Case ch Case "!"c - If Peep(2,c2) Then + If Peep(2, c2) Then Select Case c2 Case "-"c - If NextIs(3,"-"c) Then + If NextIs(3, "-"c) Then Return XmlMakeBeginCommentToken(leadingTrivia, s_scanNoTriviaFunc) End If Case "["c - If NextAre(3,"CDATA[") Then + If NextAre(3, "CDATA[") Then Return XmlMakeBeginCDataToken(leadingTrivia, s_scanNoTriviaFunc) End If Case "D"c - If Nextare(3,"OCTYPE") Then + If NextAre(3, "OCTYPE") Then Return XmlMakeBeginDTDToken(leadingTrivia) End If End Select End If Return XmlLessThanExclamationToken(state, leadingTrivia) Case "%"c - If NextIs(2,"="c) Then + If NextIs(2, "="c) Then Return XmlMakeBeginEmbeddedToken(leadingTrivia) End If Case "?"c @@ -148,36 +148,36 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeLessToken(leadingTrivia) Case "?"c - If NextIs(1,">"c) Then + If NextIs(1, ">"c) Then ' // Create token for the '?>' termination sequence Return XmlMakeEndProcessingInstructionToken(leadingTrivia) - End If + End If - Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) + Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) - Case "("c - Return XmlMakeLeftParenToken(leadingTrivia) + Case "("c + Return XmlMakeLeftParenToken(leadingTrivia) - Case ")"c - Return XmlMakeRightParenToken(leadingTrivia) + Case ")"c + Return XmlMakeRightParenToken(leadingTrivia) - Case "!"c, ";"c, "#"c, ","c, "}"c - Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) + Case "!"c, ";"c, "#"c, ","c, "}"c + Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar) - Case ":"c - Return XmlMakeColonToken(leadingTrivia) + Case ":"c + Return XmlMakeColonToken(leadingTrivia) - Case "["c - Return XmlMakeOpenBracketToken(state, leadingTrivia) + Case "["c + Return XmlMakeOpenBracketToken(state, leadingTrivia) - Case "]"c - Return XmlMakeCloseBracketToken(state, leadingTrivia) + Case "]"c + Return XmlMakeCloseBracketToken(state, leadingTrivia) - Case Else - ' // Because of weak scanning of QName, this state must always handle - ' // '=' | '\'' | '"'| '/' | '>' | '<' | '?' + Case Else + ' // Because of weak scanning of QName, this state must always handle + ' // '=' | '\'' | '"'| '/' | '>' | '<' | '?' - Return ScanXmlNcName(leadingTrivia) + Return ScanXmlNcName(leadingTrivia) End Select End While @@ -374,97 +374,97 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return XmlMakeLessToken(precedingTrivia) Case "]"c - If NextAre(Here+1,"]>") Then + If NextAre(Here + 1, "]>") Then ' // If valid characters found then return them. If Here <> 0 Then - Return XmlMakeTextLiteralToken(Nothing, Here, scratch) - End If - - ' // Create an invalid character data token for the illegal ']]>' sequence - Return XmlMakeTextLiteralToken(Nothing, 3, ERRID.ERR_XmlEndCDataNotAllowedInContent) - End If - GoTo ScanChars - - Case "#"c - ' // Even though # is valid in content, abort xml scanning if the m_State shows and error - ' // and the line begins with NL WS* # WS* KW - - 'TODO: error recovery - how can we do ths? - - 'If m_State.m_IsXmlError Then - ' MakeXmlCharToken(tokens.tkXmlCharData, Here - m_InputStreamPosition, IsAllWhitespace) - ' m_InputStreamPosition = Here - - ' Dim sharp As Token = MakeToken(tokens.tkSharp, 1) - ' m_InputStreamPosition += 1 - - ' While (m_InputStream(m_InputStreamPosition) = " "c OrElse m_InputStream(m_InputStreamPosition) = CHARACTER_TABULATION) - ' m_InputStreamPosition += 1 - ' End While - - ' ScanXmlQName() - - ' Dim restart As Token = CheckXmlForStatement() - - ' If restart IsNot Nothing Then - ' ' // Abort Xml - Found Keyword space at the beginning of the line - ' AbandonTokens(restart) - ' m_State.Init(LexicalState.VB) - ' MakeToken(tokens.tkXmlAbort, 0) - ' Return - ' End If - - ' AbandonTokens(sharp) - ' Here = m_InputStreamPosition - 'End If - GoTo ScanChars + Return XmlMakeTextLiteralToken(Nothing, Here, scratch) + End If - Case "%"c - - 'TODO: error recovery. We cannot do this. - 'If there is all whitespace after ">", it will be scanned as insignificant, - 'but in this case it is significant. - 'Also as far as I can see Dev10 does not resync on "%>" text anyways. - - '' // Even though %> is valid in pcdata. When inside of an embedded expression - '' // return this sequence separately so that the xml literal completion code can - '' // easily detect the end of an embedded expression that may be temporarily hidden - '' // by a new element. i.e. <%= %> - - 'If CanGetCharAtOffset(Here + 1) AndAlso _ - ' PeekAheadChar(Here + 1) = ">"c Then - - ' ' // If valid characters found then return them. - ' If Here <> 0 Then - ' Return XmlMakeCharDataToken(Nothing, Here, New String(value.ToArray)) - ' End If - - ' ' // Create a special pcdata token for the possible tkEndXmlEmbedded - ' Return XmlMakeCharDataToken(Nothing, 2, "%>") - 'Else - ' IsAllWhitespace = False - ' value.Add("%"c) - ' Here += 1 - 'End If - 'Continue While - GoTo ScanChars - Case Else + ' // Create an invalid character data token for the illegal ']]>' sequence + Return XmlMakeTextLiteralToken(Nothing, 3, ERRID.ERR_XmlEndCDataNotAllowedInContent) + End If + GoTo ScanChars + + Case "#"c + ' // Even though # is valid in content, abort xml scanning if the m_State shows and error + ' // and the line begins with NL WS* # WS* KW + + 'TODO: error recovery - how can we do ths? + + 'If m_State.m_IsXmlError Then + ' MakeXmlCharToken(tokens.tkXmlCharData, Here - m_InputStreamPosition, IsAllWhitespace) + ' m_InputStreamPosition = Here + + ' Dim sharp As Token = MakeToken(tokens.tkSharp, 1) + ' m_InputStreamPosition += 1 + + ' While (m_InputStream(m_InputStreamPosition) = " "c OrElse m_InputStream(m_InputStreamPosition) = CHARACTER_TABULATION) + ' m_InputStreamPosition += 1 + ' End While + + ' ScanXmlQName() + + ' Dim restart As Token = CheckXmlForStatement() + + ' If restart IsNot Nothing Then + ' ' // Abort Xml - Found Keyword space at the beginning of the line + ' AbandonTokens(restart) + ' m_State.Init(LexicalState.VB) + ' MakeToken(tokens.tkXmlAbort, 0) + ' Return + ' End If + + ' AbandonTokens(sharp) + ' Here = m_InputStreamPosition + 'End If + GoTo ScanChars + + Case "%"c + + 'TODO: error recovery. We cannot do this. + 'If there is all whitespace after ">", it will be scanned as insignificant, + 'but in this case it is significant. + 'Also as far as I can see Dev10 does not resync on "%>" text anyways. + + '' // Even though %> is valid in pcdata. When inside of an embedded expression + '' // return this sequence separately so that the xml literal completion code can + '' // easily detect the end of an embedded expression that may be temporarily hidden + '' // by a new element. i.e. <%= %> + + 'If CanGetCharAtOffset(Here + 1) AndAlso _ + ' PeekAheadChar(Here + 1) = ">"c Then + + ' ' // If valid characters found then return them. + ' If Here <> 0 Then + ' Return XmlMakeCharDataToken(Nothing, Here, New String(value.ToArray)) + ' End If + + ' ' // Create a special pcdata token for the possible tkEndXmlEmbedded + ' Return XmlMakeCharDataToken(Nothing, 2, "%>") + 'Else + ' IsAllWhitespace = False + ' value.Add("%"c) + ' Here += 1 + 'End If + 'Continue While + GoTo ScanChars + Case Else ScanChars: - ' // Check characters are valid - IsAllWhitespace = False - Dim xmlCh = ScanXmlChar(Here) - - If xmlCh.Length = 0 Then - ' bad char - If Here > 0 Then - Return XmlMakeTextLiteralToken(Nothing, Here, scratch) - Else - Return XmlMakeBadToken(Nothing, 1, ERRID.ERR_IllegalChar) + ' // Check characters are valid + IsAllWhitespace = False + Dim xmlCh = ScanXmlChar(Here) + + If xmlCh.Length = 0 Then + ' bad char + If Here > 0 Then + Return XmlMakeTextLiteralToken(Nothing, Here, scratch) + Else + Return XmlMakeBadToken(Nothing, 1, ERRID.ERR_IllegalChar) + End If End If - End If - xmlCh.AppendTo(scratch) - Here += xmlCh.Length + xmlCh.AppendTo(scratch) + Here += xmlCh.Length End Select End While @@ -497,58 +497,58 @@ ScanChars: Return XmlMakeCommentToken(precedingTrivia, Here + LengthOfLineBreak(c, Here)) Case "-"c - If NextIs(Here + 1,"-"c) Then + If NextIs(Here + 1, "-"c) Then ' // --> terminates an Xml comment but otherwise -- is an illegal character sequence. ' // The scanner will always returns "--" as a separate comment data string and the ' // the semantics will error if '--' is ever found. ' // Return valid characters up to the -- If Here > 0 Then - Return XmlMakeCommentToken(precedingTrivia, Here) - End If + Return XmlMakeCommentToken(precedingTrivia, Here) + End If - If Peep(Here + 2, c) Then - Here += 2 - ' // if > is not found then this is an error. Return the -- string + If Peep(Here + 2, c) Then + Here += 2 + ' // if > is not found then this is an error. Return the -- string - If c <> ">"c Then - Return XmlMakeCommentToken(precedingTrivia, 2) + If c <> ">"c Then + Return XmlMakeCommentToken(precedingTrivia, 2) - ' TODO: we cannot do the following + ' TODO: we cannot do the following - ' // For better error recovery, allow -> to terminate the comment. - ' // This works because the -> terminates only when the invalid -- - ' // is returned. + ' // For better error recovery, allow -> to terminate the comment. + ' // This works because the -> terminates only when the invalid -- + ' // is returned. - 'If Here + 1 < m_InputStreamEnd AndAlso _ - ' m_InputStream(Here) = "-"c AndAlso _ - ' m_InputStream(Here + 1) = ">"c Then + 'If Here + 1 < m_InputStreamEnd AndAlso _ + ' m_InputStream(Here) = "-"c AndAlso _ + ' m_InputStream(Here + 1) = ">"c Then - ' Here += 1 - 'Else - ' Continue While - 'End If - Else - Return XmlMakeEndCommentToken(precedingTrivia) + ' Here += 1 + 'Else + ' Continue While + 'End If + Else + Return XmlMakeEndCommentToken(precedingTrivia) + End If End If End If - End If - GoTo ScanChars + GoTo ScanChars - Case Else + Case Else ScanChars: - Dim xmlCh = ScanXmlChar(Here) - If xmlCh.Length <> 0 Then - Here += xmlCh.Length - Continue While - End If + Dim xmlCh = ScanXmlChar(Here) + If xmlCh.Length <> 0 Then + Here += xmlCh.Length + Continue While + End If - ' bad char - If Here > 0 Then - Return XmlMakeCommentToken(precedingTrivia, Here) - Else - Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalChar) - End If + ' bad char + If Here > 0 Then + Return XmlMakeCommentToken(precedingTrivia, Here) + Else + Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalChar) + End If End Select End While @@ -586,7 +586,7 @@ ScanChars: Return XmlMakeCDataToken(precedingTrivia, Here, scratch) Case "]"c - If NextAre(Here+1,"]>") Then + If NextAre(Here + 1, "]>") Then '// If valid characters found then return them. If Here <> 0 Then @@ -785,7 +785,7 @@ CleanUp: Dim Here = 0 Dim scratch = GetScratch() - While Peep(Here,c) + While Peep(Here, c) Select Case (c) Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION If Here > 0 Then @@ -908,7 +908,7 @@ ScanChars: GoTo CleanUp Else ' report unexpected <%= in a special way. - If NextAre(1,"%=") Then + If NextAre(1, "%=") Then Dim errEmbedStart = XmlMakeAttributeDataToken(precedingTrivia, 3, "<%=") Dim errEmberinfo = ErrorFactory.ErrorInfo(ERRID.ERR_QuotedEmbeddedExpression) result = DirectCast(errEmbedStart.SetDiagnostics({errEmberinfo}), SyntaxToken) @@ -1017,7 +1017,7 @@ CleanUp: Private Function ScanXmlChar(Here As Integer) As XmlCharResult Debug.Assert(Here >= 0) Dim c As Char - Dim ok =Peep(Here, c) + Dim ok = Peep(Here, c) Debug.Assert(ok) If Not isValidUtf16(c) Then @@ -1112,11 +1112,11 @@ CreateNCNameToken: End Function Private Function ScanXmlReference(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As XmlTextTokenSyntax - Debug.Assert( NextIs(0,"&"c)) + Debug.Assert(NextIs(0, "&"c)) Dim c, c1 As Char ' skip 1 char for "&" If Peep(1, c) Then - Select Case (c) + Select Case (c) Case "#"c Dim Here = 2 ' skip "&#" Dim result = ScanXmlCharRef(Here) @@ -1141,7 +1141,7 @@ CreateNCNameToken: Case "a"c ' // & ' // ' - If Peep(4, c1) AndAlso NextAre(2,"mp") Then + If Peep(4, c1) AndAlso NextAre(2, "mp") Then If c1 = ";"c Then Return XmlMakeAmpLiteralToken(precedingTrivia) Else @@ -1150,7 +1150,7 @@ CreateNCNameToken: Return DirectCast(noSemicolon.SetDiagnostics({noSemicolonError}), XmlTextTokenSyntax) End If - ElseIf Peep(5, c1) AndAlso NextAre(2,"pos") Then + ElseIf Peep(5, c1) AndAlso NextAre(2, "pos") Then If c1 = ";"c Then Return XmlMakeAposLiteralToken(precedingTrivia) Else @@ -1162,7 +1162,7 @@ CreateNCNameToken: Case "l"c ' // < - If Peep(3, c1) AndAlso NextIs(2,"t"c) Then + If Peep(3, c1) AndAlso NextIs(2, "t"c) Then If c1 = ";"c Then Return XmlMakeLtLiteralToken(precedingTrivia) Else @@ -1174,7 +1174,7 @@ CreateNCNameToken: Case "g"c ' // > - If Peep(3, c1) AndAlso NextIs(2,"t"c) Then + If Peep(3, c1) AndAlso NextIs(2, "t"c) Then If c1 = ";"c Then Return XmlMakeGtLiteralToken(precedingTrivia) Else @@ -1186,7 +1186,7 @@ CreateNCNameToken: Case "q"c ' // " - If Peep(5, c1) AndAlso NextAre(2,"uot") Then + If Peep(5, c1) AndAlso NextAre(2, "uot") Then If c1 = ";"c Then Return XmlMakeQuotLiteralToken(precedingTrivia) Else @@ -1249,5 +1249,7 @@ CreateNCNameToken: End If Return Nothing End Function + End Class + End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb b/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb index 7080f6ad81ff8..d6af5c85865ec 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb @@ -1,7 +1,7 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. '----------------------------------------------------------------------------- -' Contains the definition of the Scanner, which produces tokens from text +' Contains the definition of the Scanner, which produces tokens from text '----------------------------------------------------------------------------- Option Compare Binary Option Strict On @@ -151,8 +151,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Gets current reusable syntax node. - ''' If node is returned its start will be aligned with the start of current token. - ''' NOTE: Line offset may not match start of current token because of lookahead. + ''' If node is returned its start will be aligned with the start of current token. + ''' NOTE: Line offset may not match start of current token because of lookahead. ''' Friend Overridable Function GetCurrentSyntaxNode() As VisualBasicSyntaxNode Return Nothing @@ -161,12 +161,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Indicates that previously returned node has been consumed ''' and scanner needs to advance by the size of the node. - ''' + ''' ''' NOTE: the advancement is done relative to the start of the current token. - ''' Line offset may not match start of current token because of lookahead. - ''' - ''' This operation will discard lookahead tokens and reset preprocessor state - ''' to the state of current token. + ''' Line offset may not match start of current token because of lookahead. + ''' + ''' This operation will discard lookahead tokens and reset preprocessor state + ''' to the state of current token. ''' Friend Overridable Sub MoveToNextSyntaxNode() ' do not use prev token after consuming a nonterminal @@ -180,14 +180,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' ''' Indicates that previously returned node has been consumed ''' and scanner needs to advance by the size of the node. - ''' + ''' ''' NOTE: the advancement is done relative to the _lineBufferOffset. ''' Line offset will likely not match start of current token because this operation ''' is done while constructing the content of current token. - ''' + ''' ''' NOTE: This operation assumes that there is no tokens read ahead. - ''' - ''' NOTE: This operation does not change preprocessor state. + ''' + ''' NOTE: This operation does not change preprocessor state. ''' The assumption is that it is responsibility of the node consumer to update preprocessor ''' state if needed when using nodes that change preprocessor state. ''' @@ -236,7 +236,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax If state <> _currentToken.State Then ' this is a special case for switching from VB to Xml - ' we need to keep preceding trivia as it was scanned + ' we need to keep preceding trivia as it was scanned If _currentToken.State = ScannerState.VB AndAlso state = ScannerState.Content Then Dim vbTk = GetCurrentToken() diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb index 37d43c57a4076..54af92bbe03e4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb @@ -307,6 +307,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Case "]"c If NextAre(Here + 1, "]>") Then + ' // If valid characters found then return them. If Here <> 0 Then Return XmlMakeTextLiteralToken(precedingTrivia, Here, scratch) From a58d3f719babc267ccfb39b1292a08ba6e418666 Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Sun, 31 May 2015 13:25:06 +0100 Subject: [PATCH 7/9] [VB: Compiler SymbolDisplay] Add the parameter name to a ArgumentNullException. Use of string interpolation. --- .../Portable/SymbolDisplay/ObjectDisplay.vb | 2 +- .../SymbolDisplayVisitor.Members.vb | 2 +- .../SymbolDisplay/SymbolDisplayVisitor.vb | 18 +++++++++--------- .../SymbolDisplayVisitor_Minimal.vb | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/SymbolDisplay/ObjectDisplay.vb b/src/Compilers/VisualBasic/Portable/SymbolDisplay/ObjectDisplay.vb index c20e599568c55..5d6e56255cc81 100644 --- a/src/Compilers/VisualBasic/Portable/SymbolDisplay/ObjectDisplay.vb +++ b/src/Compilers/VisualBasic/Portable/SymbolDisplay/ObjectDisplay.vb @@ -126,7 +126,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay ValidateOptions(options) If value Is Nothing Then - Throw New ArgumentNullException() + Throw New ArgumentNullException( NameOf(value)) End If Dim pooledBuilder = PooledStringBuilder.GetInstance() diff --git a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.Members.vb b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.Members.vb index ecd35722b0d3a..0fd6b6d5f92cc 100644 --- a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.Members.vb +++ b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.Members.vb @@ -346,7 +346,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Case MethodKind.AnonymousFunction ' there is no name to show, but it must be handled to not cause - ' the NYI below. + ' the NYI below. Case Else Throw ExceptionUtilities.UnexpectedValue(symbol.MethodKind) diff --git a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.vb b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.vb index e2f0a8f926fe4..e240e580a4f68 100644 --- a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.vb +++ b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor.vb @@ -11,10 +11,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Private ReadOnly _escapeKeywordIdentifiers As Boolean - ' A Symbol in VB might be a PENamedSymbolWithEmittedNamespaceName and in that case the + ' A Symbol in VB might be a PENamedSymbolWithEmittedNamespaceName and in that case the ' casing of the contained types and namespaces might differ because of merged classes/namespaces. - ' To maintain the original spelling an emittedNamespaceName with the correct spelling is passed to - ' this visitor. + ' To maintain the original spelling an emittedNamespaceName with the correct spelling is passed to + ' this visitor. Friend Sub New( builder As ArrayBuilder(Of SymbolDisplayPart), @@ -102,9 +102,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Private Function EscapeIdentifier(identifier As String) As String - ' always escape keywords, + ' always escape keywords, If SyntaxFacts.GetKeywordKind(identifier) <> SyntaxKind.None Then - Return String.Format("[{0}]", identifier) + Return $"[{identifier}]" End If ' The minimal flag is e.g. used by the service layer when generating code (example: simplify name). @@ -113,13 +113,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic If Me.IsMinimizing Then Dim contextualKeywordKind As SyntaxKind = SyntaxFacts.GetContextualKeywordKind(identifier) - ' Leading implicit line continuation is allowed before query operators (Aggregate, Distinct, From, Group By, - ' Group Join, Join, Let, Order By, Select, Skip, Skip While, Take, Take While, Where, In, Into, On, Ascending, + ' Leading implicit line continuation is allowed before query operators (Aggregate, Distinct, From, Group By, + ' Group Join, Join, Let, Order By, Select, Skip, Skip While, Take, Take While, Where, In, Into, On, Ascending, ' and Descending). ' If the current identifier is one of these keywords, we need to escape them to avoid parsing issues if the ' previous line was a query expression. ' - ' In addition to the query operators, we need to escape the identifier "preserve" to avoid ambiguities + ' In addition to the query operators, we need to escape the identifier "preserve" to avoid ambiguities ' inside of a dim statement (dim [preserve].ArrayName(1)) Select Case contextualKeywordKind Case SyntaxKind.AggregateKeyword, @@ -139,7 +139,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic SyntaxKind.AscendingKeyword, SyntaxKind.DescendingKeyword, SyntaxKind.PreserveKeyword - Return String.Format("[{0}]", identifier) + Return $"[{identifier}]" End Select End If diff --git a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb index 0d6f65a80c7f4..2bcd96e563d3f 100644 --- a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb +++ b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb @@ -16,7 +16,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Debug.Assert(symbol.ContainingNamespace IsNot Nothing OrElse symbol.IsGlobalNamespace) ' NOTE(cyrusn): We only call this once we've already checked if there is an alias that - ' corresponds to this namespace. + ' corresponds to this namespace. If symbol.IsGlobalNamespace Then ' nothing to add for global namespace itself From 711246e8073017f52fcfc6e0312b2f530ff63659 Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Tue, 2 Jun 2015 02:05:31 +0100 Subject: [PATCH 8/9] [VB Compiler Scanner] Rename method so it does't use contraction. Remove Inlining Attribute. Restructured and IF statement in ScanTokenCommon --- .../Portable/Scanner/CharacterInfo.vb | 6 +++--- .../VisualBasic/Portable/Scanner/Scanner.vb | 18 +++++++++--------- .../Portable/Scanner/ScannerBuffer.vb | 2 -- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb index 1f6f7bc8b621e..159ab2da46bf4 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb @@ -506,17 +506,17 @@ Namespace Microsoft.CodeAnalysis.VisualBasic End Function - Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char) As Boolean + Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) End Function - Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean + Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) End Function - Friend Function IsntFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean + Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) AndAlso (c <> c3) End Function diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index 9a82b06d6766a..edede0d5e1930 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -597,7 +597,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ' optimization for a common case ' the ASCII range between ': and ~ , with exception of except "'", "_" and R cannot start trivia - If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsntFrom("'"c, "_"c, "R"c, "r"c) Then + If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsnotFrom("'"c, "_"c, "R"c, "r"c) Then Return Nothing End If @@ -1141,14 +1141,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Debug.Assert(Not IsNewLine(ch)) If fullWidth Then Debug.Assert(Not IsDoubleQuote(ch)) - Else - If IsDoubleQuote(ch) Then + Return nothing + End If + If IsDoubleQuote(ch) Then Return ScanStringLiteral(precedingTrivia) - End If - If IsFullWidth(ch) Then - ch = MakeHalfWidth(ch) - Return ScanTokenFullWidth(precedingTrivia, ch) - End If + End If + If IsFullWidth(ch) Then + ch = MakeHalfWidth(ch) + Return ScanTokenFullWidth(precedingTrivia, ch) End If Return Nothing End Function @@ -1262,7 +1262,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax length += 1 If Peep(length, c) Then 'if the second "<" is a part of "<%" - like in "<<%" , we do not want to use it. - If c.IsntFrom("%"c, FULLWIDTH_PERCENT_SIGN) Then + If c.IsnotFrom("%"c, FULLWIDTH_PERCENT_SIGN) Then If TrySkipFollowingEquals(length) Then Return MakeLessThanLessThanEqualsToken(precedingTrivia, length) Else diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb index 9d63044c5bca4..5a8ae75621b38 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb @@ -105,7 +105,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' NOTE: This is different behaviour to the Try methods that always return a value. ''' ''' Returns True if it possible to peek at, otherwise returns False. - Friend Function Peep(at As Integer, ByRef ch As Char) As Boolean ' CanGet(at) Debug.Assert(_lineBufferOffset + at >= 0) @@ -132,7 +131,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax ''' NOTE: This is different behaviour to the Try methods that always return a value. ''' ''' Returns True if it possible to peek at, otherwise returns False. - Friend Function Peep(ByRef ch As Char) As Boolean ' CanGet() If _lineBufferOffset >= _bufferLen Then From 1bb3643f0e69c24203adf2c9f6cbf0e1e3868385 Mon Sep 17 00:00:00 2001 From: AdamSpeight2008 Date: Tue, 2 Jun 2015 02:19:04 +0100 Subject: [PATCH 9/9] [VB Compiler Scanner] Remove Inlining Attribute from some methods --- src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb index edede0d5e1930..10885abcaec81 100644 --- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb +++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb @@ -397,12 +397,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax #Region "Buffer helpers" - Private Function NextAre(chars As String) As Boolean Return NextAre(0, chars) End Function - Private Function NextAre(offset As Integer, chars As String) As Boolean Debug.Assert(Not String.IsNullOrEmpty(chars)) Dim n = chars.Length @@ -413,17 +411,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax Return True End Function - Private Function NextIs(offset As Integer, c As Char) As Boolean Return CanGet(offset) AndAlso (Peek(offset) = c) End Function - Private Function CanGet() As Boolean Return _lineBufferOffset < _bufferLen End Function - Private Function CanGet(num As Integer) As Boolean Debug.Assert(_lineBufferOffset + num >= 0) Debug.Assert(num >= -MaxCharsLookBehind)