diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb b/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb
index e01a47a99bf26..dc241af8f0957 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/Blender.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Compare Binary
Option Strict On
@@ -139,7 +139,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return node.GetTrailingTrivia().Any(SyntaxKind.EndOfLineTrivia)
Case SyntaxKind.SingleLineIfStatement,
SyntaxKind.SingleLineElseClause
- ' Steer clear of single-line if's because they they have custom handling of statement
+ ' Steer clear of single-line if's because they they have custom handling of statement
' terminators that may make it difficult to reuse sub-statements.
Return False
Case Else
@@ -301,7 +301,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Shared Function ShouldCrumble(node As VisualBasicSyntaxNode) As Boolean
If TypeOf node Is StructuredTriviaSyntax Then
' Do not crumble into structured trivia content.
- ' we will not use any of the parts anyways and
+ ' we will not use any of the parts anyways and
' evaluation of directives may go out of sync.
Return False
End If
@@ -516,14 +516,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
_currentToken = _currentToken.With(nextPreprocessorState)
End If
- ' this will discard any prefetched tokens, including current.
+ ' this will discard any prefetched tokens, including current.
' We do not need them since we moved to completely new node.
MyBase.MoveToNextSyntaxNode()
TryPopNode()
' at this point, all three pointers (position in old tree, position in text, position in parser)
- ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState)
+ ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState)
' should point to same position (in sync)
End Sub
@@ -537,7 +537,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' just move forward
_lineBufferOffset = _lineBufferOffset + _curNodeLength
- ' this will just verify that we do not have any prefetched tokens, including current.
+ ' this will just verify that we do not have any prefetched tokens, including current.
' otherwise advancing linebuffer offeset could go out of sync with token stream.
MyBase.MoveToNextSyntaxNodeInTrivia()
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb
index 099a979f87b7c..159ab2da46bf4 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/CharacterInfo.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Strict On
Option Explicit On
@@ -339,7 +339,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
End Function
'''
- ''' Creates a half width form Unicode character string.
+ ''' Creates a half width form Unicode character string.
'''
''' The text representing the original identifier. This can be in full width or half width Unicode form.
''' A string representing the text in a half width Unicode form.
@@ -488,4 +488,38 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
End Function
End Class
+ Friend Module CharExts
+
+
+ Friend Function IsFrom(c As Char, c0 As Char, c1 As Char) As Boolean
+ Return (c = c0) OrElse (c = c1)
+ End Function
+
+
+ Friend Function IsFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean
+ Return (c = c0) OrElse (c = c1) OrElse (c = c2)
+ End Function
+
+
+ Friend Function IsFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean
+ Return (c = c0) OrElse (c = c1) OrElse (c = c2) OrElse (c = c3)
+ End Function
+
+
+ Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char) As Boolean
+ Return (c <> c0) AndAlso (c <> c1)
+ End Function
+
+
+ Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char, c2 As Char) As Boolean
+ Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2)
+ End Function
+
+
+ Friend Function IsnotFrom(c As Char, c0 As Char, c1 As Char, c2 As Char, c3 As Char) As Boolean
+ Return (c <> c0) AndAlso (c <> c1) AndAlso (c <> c2) AndAlso (c <> c3)
+ End Function
+
+ End Module
+
End Namespace
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb b/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb
index 29ef1692c75c2..f8aaa56b2c5eb 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/Directives.vb
@@ -31,8 +31,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim restorePoint = CreateRestorePoint()
Me._isScanningDirective = True
- ' since we do not have lookahead tokens, this just
- ' resets current token to _lineBufferOffset
+ ' since we do not have lookahead tokens, this just
+ ' resets current token to _lineBufferOffset
Me.GetNextTokenInState(ScannerState.VB)
Dim currentNonterminal = Me.GetCurrentSyntaxNode()
@@ -43,7 +43,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Me.MoveToNextSyntaxNodeInTrivia()
' adjust current token to just after the node
- ' we need that in case we need to skip some disabled text
+ ' we need that in case we need to skip some disabled text
'(yes we do tokenize disabled text for compatibility reasons)
Me.GetNextTokenInState(ScannerState.VB)
@@ -91,10 +91,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
disabledCode = SkipConditionalCompilationSection()
End If
- ' Here we add the directive and disabled text that follows it
+ ' Here we add the directive and disabled text that follows it
' (if there is any) to the trivia list
- ' processing statement could add an error to it,
+ ' processing statement could add an error to it,
' so we may need to rebuild the trivia node
If statement IsNot directiveTrivia Then
directiveTrivia = statement
@@ -507,7 +507,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'//
'// Skip all text until the end of the current conditional section. This will also return the
'// span of text that was skipped
- '//
+ '//
'//-------------------------------------------------------------------------------------------------
Private Function SkipConditionalCompilationSection() As SyntaxList(Of VisualBasicSyntaxNode)
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb
index 47997330eb86d..bd05eb43d7068 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/KeywordTable.vb
@@ -269,7 +269,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax
Public kdIsQueryClause As Boolean
'// Note: "CanFollowExpr" says whether this token can come after an expression.
- '// e.g. "Dim x = From i In Select i" is valid, therefore "Select" can come follow an expression.
+ '// e.g. "Dim x = From i In Select i" is valid, therefore "Select" can come follow an expression.
'// The complete list was discovered through the tool in vb\Language\Tools\VBGrammarAnalyzer\vbgrammar.html
'// If you add keywords, then make sure that they're added to the official language grammar, and re-run the tool.
Public kdCanFollowExpr As Boolean
@@ -304,9 +304,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax
Private Shared Function EnsureHalfWidth(s As String) As String
Dim result As Char() = Nothing
-
For i As Integer = 0 To s.Length - 1
- Dim ch = s(i)
+ Dim ch As Char = s(i)
If SyntaxFacts.IsFullWidth(ch) Then
ch = SyntaxFacts.MakeHalfWidth(ch)
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb b/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb
index 298c37f2058d5..32a0f8819a232 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb
@@ -16,16 +16,16 @@ Imports Microsoft.CodeAnalysis.VisualBasic.Syntax
Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'' The QuickTokenAccumulator is a small mini-tokenizer that may fail. It consumes characters and
'' eventually either decides that either it found a complete token (including the trivia on either
- '' side), or it gives up and says that the full scanner should take an attempt. It also accumulates the
+ '' side), or it gives up and says that the full scanner should take an attempt. It also accumulates the
'' token into a character buffer and computes a hash code. The entire tokenization is done by a single
'' routine without any memory allocations to keep it very fast.
- ''
+ ''
'' Currently it only handles two cases:
'' optional-whitespace keyword-or-identifier optional-whitespace
'' optional-whitespace single-char-punctuation optional-whitespace
- ''
+ ''
'' where the whitespace does not include newlines.
- '
+ ''
'' The VB tokenization rules are complex, and care needs to be taken in constructing the quick
'' tokenization. For example "REM" begins a comment, so it can't be tokenized as a keyword or identifier.
'' Similar problems arise with multi-character punctuation tokens, which can have embedded spaces.
@@ -63,7 +63,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'TODO: why : and ; are complex? (8th row, 3 and 4)
- ' The following table classifies the first &H180 Unicode characters.
+ ' The following table classifies the first &H180 Unicode characters.
' R and r are marked as COMPLEX so that quick-scanning doesn't stop after "REM".
' # is marked complex as it may start directives.
' PERF: Use UShort instead of CharFlags so the compiler can use array literal initialization.
@@ -163,7 +163,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' Attempt to scan a single token.
' If it succeeds, return True, and the characters, length, and hashcode of the token
- ' can be retrieved by other functions.
+ ' can be retrieved by other functions.
' If it fails (the token is too complex), return False.
Public Function QuickScanToken(allowLeadingMultilineTrivia As Boolean) As QuickScanResult
Dim state As AccumulatorState = If(allowLeadingMultilineTrivia, AccumulatorState.InitialAllowLeadingMultilineTrivia, AccumulatorState.Initial)
@@ -330,7 +330,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
index += 1
- 'FNV-like hash should work here
+ 'FNV-like hash should work here
'since these strings are short and mostly ASCII
hashCode = (hashCode Xor unicodeValue) * Hash.FnvPrime
End While
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb
index b669b950320a3..10885abcaec81 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Compare Binary
@@ -37,8 +37,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private ReadOnly _sbPooled As PooledStringBuilder = PooledStringBuilder.GetInstance
'''
- ''' DO NOT USE DIRECTLY.
- ''' USE GetScratch()
+ ''' DO NOT USE DIRECTLY.
+ ''' USE GetScratch()
'''
Private ReadOnly _sb As StringBuilder = _sbPooled.Builder
Private ReadOnly _triviaListPool As New SyntaxListPool
@@ -90,7 +90,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function GetScratch() As StringBuilder
' the normal pattern is that we clean scratch after use.
- ' hitting this assert very likely indicates that you
+ ' hitting this assert very likely indicates that you
' did not release scratch content or worse trying to use
' scratch in two places at a time.
Debug.Assert(_sb.Length = 0, "trying to use dirty buffer?")
@@ -157,7 +157,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function GetNextToken(Optional allowLeadingMultilineTrivia As Boolean = False) As SyntaxToken
- ' Use quick token scanning to see if we can scan a token quickly.
+ ' Use quick token scanning to see if we can scan a token quickly.
Dim quickToken = QuickScanToken(allowLeadingMultilineTrivia)
If quickToken.Succeeded Then
@@ -225,16 +225,16 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function ScanNextCharAsToken(leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
Dim token As SyntaxToken
-
- If Not CanGet() Then
+ Dim c As Char
+ If Not Peep(c) Then
token = MakeEofToken(leadingTrivia)
Else
_badTokenCount += 1
If _badTokenCount < BadTokenCountLimit Then
' // Don't break up surrogate pairs
- Dim c = Peek()
- Dim length = If(IsHighSurrogate(c) AndAlso CanGet(1) AndAlso IsLowSurrogate(Peek(1)), 2, 1)
+ c = Peek()
+ Dim length = If(IsHighSurrogate(c) AndAlso Peep(1, c) AndAlso IsLowSurrogate(c), 2, 1)
token = MakeBadToken(leadingTrivia, length, ERRID.ERR_IllegalChar)
Else
' If we get too many characters that we cannot make sense of, absorb the rest of the input.
@@ -265,9 +265,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End If
Dim condLineStart = _lineBufferOffset
-
- While (CanGet())
- Dim c As Char = Peek()
+ Dim c As Char
+ While Peep(c)
Select Case (c)
@@ -281,12 +280,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
EatWhitespace()
Continue While
- Case _
- "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c,
- "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c,
- "y"c, "z"c, "A"c, "B"c, "C"c, "D"c, "E"c, "F"c, "G"c, "H"c, "I"c, "J"c,
- "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, "R"c, "S"c, "T"c, "U"c, "V"c,
- "W"c, "X"c, "Y"c, "Z"c, "'"c, "_"c
+ Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c,
+ "m"c, "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c,
+ "y"c, "z"c, "A"c, "B"c, "C"c, "D"c, "E"c, "F"c, "G"c, "H"c, "I"c, "J"c,
+ "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c, "R"c, "S"c, "T"c, "U"c, "V"c,
+ "W"c, "X"c, "Y"c, "Z"c, "'"c, "_"c
EatThroughLine()
condLineStart = _lineBufferOffset
@@ -322,23 +320,22 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Sub EatThroughLine()
- While CanGet()
- Dim c As Char = Peek()
-
- If IsNewLine(c) Then
- EatThroughLineBreak(c)
- Return
- Else
- AdvanceChar()
- End If
- End While
+ Dim c As Char
+ While Peep(c)
+ If IsNewLine(c) Then
+ EatThroughLineBreak(c)
+ Return
+ Else
+ AdvanceChar()
+ End If
+ End While
End Sub
'''
''' Gets a chunk of text as a DisabledCode node.
'''
''' The range of text.
- ''' The DisabledCode node.
+ ''' The DisabledCode node.
Friend Function GetDisabledTextAt(span As TextSpan) As SyntaxTrivia
If span.Start >= 0 AndAlso span.End <= _bufferLen Then
Return SyntaxFactory.DisabledTextTrivia(GetTextNotInterned(span.Start, span.Length))
@@ -487,11 +484,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function LengthOfLineBreak(StartCharacter As Char, Optional here As Integer = 0) As Integer
- Debug.Assert(CanGet(here))
+#If DEBUG Then
+ Dim c As Char
+ Dim res = Peep(here, c)
+ Debug.Assert(res)
Debug.Assert(IsNewLine(StartCharacter))
-
- Debug.Assert(StartCharacter = Peek(here))
-
+ Debug.Assert(StartCharacter = c)
+#End If
If StartCharacter = CARRIAGE_RETURN AndAlso NextIs(here + 1, LINE_FEED) Then
Return 2
End If
@@ -503,7 +502,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'''
''' Accept a CR/LF pair or either in isolation as a newline.
''' Make it a statement separator
- '''
+ '''
Private Function ScanNewlineAsStatementTerminator(startCharacter As Char, precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
If _lineBufferOffset < _endOfTerminatorTrivia Then
Dim width = LengthOfLineBreak(startCharacter)
@@ -533,27 +532,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function ScanLineContinuation(tList As SyntaxListBuilder) As Boolean
- If Not CanGet() Then
- Return False
- End If
-
- If Not IsAfterWhitespace() Then
- Return False
- End If
-
- Dim ch As Char = Peek()
- If Not IsUnderscore(ch) Then
+ Dim ch As Char
+ If Not Peep(ch) OrElse Not IsAfterWhitespace() OrElse Not IsUnderscore(ch) Then
Return False
End If
Dim Here = 1
- While CanGet(Here)
- ch = Peek(Here)
- If IsWhitespace(ch) Then
- Here += 1
- Else
- Exit While
- End If
+ While Peep(Here, ch) AndAlso IsWhitespace(ch)
+ Here += 1
End While
' Line continuation is valid at the end of the
@@ -578,10 +564,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' do not include the new line character since that would confuse code handling
' implicit line continuations. (See Scanner::EatLineContinuation.) Otherwise,
' include the new line and any additional spaces as trivia.
- If startComment = 0 AndAlso
- CanGet(Here) AndAlso
- Not IsNewLine(Peek(Here)) Then
+ If startComment = 0 AndAlso Peep(Here, ch) AndAlso Not IsNewLine(ch) Then
tList.Add(MakeEndOfLineTrivia(GetText(newLine)))
If spaces > 0 Then
tList.Add(MakeWhiteSpaceTrivia(GetText(spaces)))
@@ -601,15 +585,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
''' Consumes all trivia until a nontrivia char is found
'''
Friend Function ScanMultilineTrivia() As SyntaxList(Of VisualBasicSyntaxNode)
- If Not CanGet() Then
+ Dim ch As Char
+ If Not Peep(ch) Then
Return Nothing
End If
- Dim ch = Peek()
-
' optimization for a common case
' the ASCII range between ': and ~ , with exception of except "'", "_" and R cannot start trivia
- If ch > ":"c AndAlso ch <= "~"c AndAlso ch <> "'"c AndAlso ch <> "_"c AndAlso ch <> "R"c AndAlso ch <> "r"c Then
+ If ch > ":"c AndAlso ch <= "~"c AndAlso ch.IsnotFrom("'"c, "_"c, "R"c, "r"c) Then
Return Nothing
End If
@@ -626,7 +609,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
''' Scans a single piece of trivia
'''
Private Function TryScanSinglePieceOfMultilineTrivia(tList As SyntaxListBuilder) As Boolean
- If CanGet() Then
+ Dim ch As Char
+ If Peep(ch) Then
Dim atNewLine = IsAtNewLine()
@@ -641,7 +625,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End If
End If
- Dim ch = Peek()
If IsWhitespace(ch) Then
' eat until linebreak or nonwhitespace
Dim wslen = GetWhitespaceLength(1)
@@ -686,11 +669,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' check for #
Private Function StartsDirective(Here As Integer) As Boolean
- If CanGet(Here) Then
- Dim ch = Peek(Here)
- Return IsHash(ch)
- End If
- Return False
+ Dim ch As Char
+ Return Peep(Here, ch) AndAlso IsHash(ch)
End Function
Private Function IsAtNewLine() As Boolean
@@ -730,8 +710,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Sub
Private Sub ScanSingleLineTriviaInXmlDoc(tList As SyntaxListBuilder)
- If CanGet() Then
- Dim c As Char = Peek()
+ Dim c As Char
+ If Peep(c) Then
Select Case (c)
' // Whitespace
' // S ::= (#x20 | #x9 | #xD | #xA)+
@@ -763,7 +743,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Sub ScanWhitespaceAndLineContinuations(tList As SyntaxListBuilder)
- If CanGet() AndAlso IsWhitespace(Peek()) Then
+ Dim c As Char
+ If Peep(c) AndAlso IsWhitespace(c) Then
tList.Add(ScanWhitespace(1))
' collect { lineCont, ws }
While ScanLineContinuation(tList)
@@ -830,12 +811,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' 3. colon followed by new line -> colon terminator + new line terminator
' 4. new line followed by new line -> new line terminator + new line terminator
- ' Case 3 is required to parse single line if's and numeric labels.
+ ' Case 3 is required to parse single line if's and numeric labels.
' Case 4 is required to limit explicit line continuations to single new line
+ Dim ch As Char
+ If Peep(ch) Then
- If CanGet() Then
-
- Dim ch As Char = Peek()
Dim startOfTerminatorTrivia = _lineBufferOffset
If IsNewLine(ch) Then
@@ -847,11 +827,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' collect { ws, colon }
Do
Dim len = GetWhitespaceLength(0)
- If Not CanGet(len) Then
+ If Not Peep(len, ch) Then
Exit Do
End If
- ch = Peek(len)
If Not IsColonAndNotColonEquals(ch, offset:=len) Then
Exit Do
End If
@@ -887,7 +866,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function GetWhitespaceLength(len As Integer) As Integer
' eat until linebreak or nonwhitespace
- While CanGet(len) AndAlso IsWhitespace(Peek(len))
+ Dim c As Char
+ While Peep(len, c) AndAlso IsWhitespace(c)
len += 1
End While
Return len
@@ -895,7 +875,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function GetXmlWhitespaceLength(len As Integer) As Integer
' eat until linebreak or nonwhitespace
- While CanGet(len) AndAlso IsXmlWhitespace(Peek(len))
+ Dim c As Char
+ While Peep(len, c) AndAlso IsXmlWhitespace(c)
len += 1
End While
Return len
@@ -918,32 +899,32 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Sub EatWhitespace()
- Debug.Assert(CanGet)
- Debug.Assert(IsWhitespace(Peek()))
-
+ Dim c As Char
+#If DEBUG Then
+ Dim ok = Peep(c)
+ Debug.Assert(ok)
+ Debug.Assert(IsWhitespace(c))
+#End If
AdvanceChar()
-
' eat until linebreak or nonwhitespace
- While CanGet() AndAlso IsWhitespace(Peek)
+ While Peep(c) AndAlso IsWhitespace(c)
AdvanceChar()
End While
End Sub
Private Function PeekStartComment(i As Integer) As Integer
-
- If CanGet(i) Then
- Dim ch = Peek(i)
-
+ Dim ch As Char
+ If Peep(i, ch) Then
If IsSingleQuote(ch) Then
Return 1
- ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso
- CanGet(i + 2) AndAlso MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso
- MatchOneOrAnotherOrFullwidth(Peek(i + 2), "M"c, "m"c) Then
+ ElseIf MatchOneOrAnotherOrFullwidth(ch, "R"c, "r"c) AndAlso Peep(i + 2, ch) AndAlso
+ MatchOneOrAnotherOrFullwidth(Peek(i + 1), "E"c, "e"c) AndAlso
+ MatchOneOrAnotherOrFullwidth(ch, "M"c, "m"c) Then
- If Not CanGet(i + 3) OrElse IsNewLine(Peek(i + 3)) Then
+ If Not Peep(i + 3, ch) OrElse IsNewLine(ch) Then
' have only 'REM'
Return 3
- ElseIf Not IsIdentifierPartCharacter(Peek(i + 3)) Then
+ ElseIf Not IsIdentifierPartCharacter(ch) Then
' have 'REM '
Return 4
End If
@@ -961,9 +942,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim looksLikeDocComment As Boolean = StartsXmlDoc(0)
' eat all chars until EoL
- While CanGet(length) AndAlso
- Not IsNewLine(Peek(length))
-
+ Dim c As Char
+ While Peep(length, c) AndAlso Not IsNewLine(c)
length += 1
End While
@@ -995,450 +975,191 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
#End Region
- ' at this point it is very likely that we are located at
- ' the beginning of a token
- Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
-
- If Not CanGet() Then
- Return MakeEofToken(precedingTrivia)
- End If
+ ' at this point it is very likely that we are located at
+ ' the beginning of a token
+ Private Function ScanTokenCommon(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char, fullWidth As Boolean) As SyntaxToken
+ Dim lengthWithMaybeEquals As Integer = 1
+ Dim c As Char
- Dim ch As Char = Peek()
Select Case ch
- Case CARRIAGE_RETURN, LINE_FEED, NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR
+ Case CARRIAGE_RETURN, LINE_FEED
Return ScanNewlineAsStatementTerminator(ch, precedingTrivia)
-
+ Case NEXT_LINE, LINE_SEPARATOR, PARAGRAPH_SEPARATOR
+ If Not fullWidth Then
+ Return ScanNewlineAsStatementTerminator(ch, precedingTrivia)
+ End If
Case " "c, CHARACTER_TABULATION, "'"c
Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch)))
Return Nothing ' trivia cannot start a token
-
Case "@"c
- Return MakeAtToken(precedingTrivia, False)
-
+ Return MakeAtToken(precedingTrivia, fullWidth)
Case "("c
- Return MakeOpenParenToken(precedingTrivia, False)
-
+ Return MakeOpenParenToken(precedingTrivia, fullWidth)
Case ")"c
- Return MakeCloseParenToken(precedingTrivia, False)
-
+ Return MakeCloseParenToken(precedingTrivia, fullWidth)
Case "{"c
- Return MakeOpenBraceToken(precedingTrivia, False)
-
+ Return MakeOpenBraceToken(precedingTrivia, fullWidth)
Case "}"c
- Return MakeCloseBraceToken(precedingTrivia, False)
-
+ Return MakeCloseBraceToken(precedingTrivia, fullWidth)
Case ","c
- Return MakeCommaToken(precedingTrivia, False)
-
+ Return MakeCommaToken(precedingTrivia, fullWidth)
Case "#"c
Dim dl = ScanDateLiteral(precedingTrivia)
- If dl IsNot Nothing Then
- Return dl
- Else
- Return MakeHashToken(precedingTrivia, False)
- End If
-
+ Return If(dl, MakeHashToken(precedingTrivia, fullWidth))
Case "&"c
- If CanGet(1) AndAlso BeginsBaseLiteral(Peek(1)) Then
+ If Peep(1, c) AndAlso BeginsBaseLiteral(c) Then
Return ScanNumericLiteral(precedingTrivia)
End If
-
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeAmpersandEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakeAmpersandToken(precedingTrivia, False)
+ Return MakeAmpersandToken(precedingTrivia, fullWidth)
End If
-
Case "="c
- Return MakeEqualsToken(precedingTrivia, False)
-
+ Return MakeEqualsToken(precedingTrivia, fullWidth)
Case "<"c
- Return ScanLeftAngleBracket(precedingTrivia, False, _scanSingleLineTriviaFunc)
-
+ Return ScanLeftAngleBracket(precedingTrivia, fullWidth, _scanSingleLineTriviaFunc)
Case ">"c
- Return ScanRightAngleBracket(precedingTrivia, False)
-
+ Return ScanRightAngleBracket(precedingTrivia, fullWidth)
Case ":"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeColonEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return ScanColonAsStatementTerminator(precedingTrivia, False)
+ Return ScanColonAsStatementTerminator(precedingTrivia, fullWidth)
End If
-
Case "+"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakePlusEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakePlusToken(precedingTrivia, False)
+ Return MakePlusToken(precedingTrivia, fullWidth)
End If
-
Case "-"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeMinusEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakeMinusToken(precedingTrivia, False)
+ Return MakeMinusToken(precedingTrivia, fullWidth)
End If
-
Case "*"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeAsteriskEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakeAsteriskToken(precedingTrivia, False)
+ Return MakeAsteriskToken(precedingTrivia, fullWidth)
End If
-
Case "/"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakeSlashToken(precedingTrivia, False)
+ Return MakeSlashToken(precedingTrivia, fullWidth)
End If
-
Case "\"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeBackSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakeBackslashToken(precedingTrivia, False)
+ Return MakeBackslashToken(precedingTrivia, fullWidth)
End If
-
Case "^"c
- Dim lengthWithMaybeEquals = 1
If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
Return MakeCaretEqualsToken(precedingTrivia, lengthWithMaybeEquals)
Else
- Return MakeCaretToken(precedingTrivia, False)
+ Return MakeCaretToken(precedingTrivia, fullWidth)
End If
-
Case "!"c
- Return MakeExclamationToken(precedingTrivia, False)
-
+ Return MakeExclamationToken(precedingTrivia, fullWidth)
Case "."c
If CanGet(1) AndAlso IsDecimalDigit(Peek(1)) Then
Return ScanNumericLiteral(precedingTrivia)
Else
- Return MakeDotToken(precedingTrivia, False)
+ Return MakeDotToken(precedingTrivia, fullWidth)
End If
-
- Case "0"c,
- "1"c,
- "2"c,
- "3"c,
- "4"c,
- "5"c,
- "6"c,
- "7"c,
- "8"c,
- "9"c
+ Case "0"c, "1"c, "2"c, "3"c, "4"c,
+ "5"c, "6"c, "7"c, "8"c, "9"c
Return ScanNumericLiteral(precedingTrivia)
-
Case """"c
Return ScanStringLiteral(precedingTrivia)
-
Case "A"c
If NextAre(1, "s ") Then
-
' TODO: do we allow widechars in keywords?
- Dim spelling = "As"
AdvanceChar(2)
- Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia)
+ Return MakeKeyword(SyntaxKind.AsKeyword, "As", precedingTrivia)
Else
Return ScanIdentifierOrKeyword(precedingTrivia)
End If
-
Case "E"c
If NextAre(1, "nd ") Then
-
' TODO: do we allow widechars in keywords?
- Dim spelling = "End"
AdvanceChar(3)
- Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia)
+ Return MakeKeyword(SyntaxKind.EndKeyword, "End", precedingTrivia)
Else
Return ScanIdentifierOrKeyword(precedingTrivia)
End If
-
Case "I"c
If NextAre(1, "f ") Then
-
' TODO: do we allow widechars in keywords?
- Dim spelling = "If"
AdvanceChar(2)
- Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia)
+ Return MakeKeyword(SyntaxKind.IfKeyword, "If", precedingTrivia)
Else
Return ScanIdentifierOrKeyword(precedingTrivia)
End If
-
Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, "m"c,
"n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, "y"c, "z"c
Return ScanIdentifierOrKeyword(precedingTrivia)
-
Case "B"c, "C"c, "D"c, "F"c, "G"c, "H"c, "J"c, "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c,
- "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c
+ "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c
Return ScanIdentifierOrKeyword(precedingTrivia)
-
Case "_"c
- If CanGet(1) AndAlso IsIdentifierPartCharacter(Peek(1)) Then
+ If Peep(1, c) AndAlso IsIdentifierPartCharacter(c) Then
Return ScanIdentifierOrKeyword(precedingTrivia)
End If
-
Dim err As ERRID = ERRID.ERR_ExpectedIdentifier
Dim len = GetWhitespaceLength(1)
If Not CanGet(len) OrElse IsNewLine(Peek(len)) OrElse PeekStartComment(len) > 0 Then
err = ERRID.ERR_LineContWithCommentOrNoPrecSpace
End If
-
' not a line continuation and cannot start identifier.
Return MakeBadToken(precedingTrivia, 1, err)
-
Case "["c
Return ScanBracketedIdentifier(precedingTrivia)
-
Case "?"c
- Return MakeQuestionToken(precedingTrivia, False)
-
+ Return MakeQuestionToken(precedingTrivia, fullWidth)
Case "%"c
- If CanGet(1) AndAlso
- Peek(1) = ">"c Then
+ If NextIs(1, ">"c) Then
Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc)
End If
-
Case "$"c, FULLWIDTH_DOLLAR_SIGN
- If CanGet(1) AndAlso IsDoubleQuote(Peek(1)) Then
+ If Not fullWidth AndAlso Peep(1, c) AndAlso IsDoubleQuote(c) Then
Return MakePunctuationToken(precedingTrivia, 2, SyntaxKind.DollarSignDoubleQuoteToken)
End If
-
End Select
-
If IsIdentifierStartCharacter(ch) Then
Return ScanIdentifierOrKeyword(precedingTrivia)
End If
-
Debug.Assert(Not IsNewLine(ch))
-
+ If fullWidth Then
+ Debug.Assert(Not IsDoubleQuote(ch))
+ Return nothing
+ End If
If IsDoubleQuote(ch) Then
- Return ScanStringLiteral(precedingTrivia)
+ Return ScanStringLiteral(precedingTrivia)
End If
-
If IsFullWidth(ch) Then
ch = MakeHalfWidth(ch)
Return ScanTokenFullWidth(precedingTrivia, ch)
End If
-
Return Nothing
End Function
- ' REVIEW: Is there a better way to reuse this logic?
- Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken
- Select Case ch
- Case CARRIAGE_RETURN, LINE_FEED
- Return ScanNewlineAsStatementTerminator(ch, precedingTrivia)
-
- Case " "c, CHARACTER_TABULATION, "'"c
- Debug.Assert(False, String.Format("Unexpected char: &H{0:x}", AscW(ch)))
- Return Nothing ' trivia cannot start a token
-
- Case "@"c
- Return MakeAtToken(precedingTrivia, True)
-
- Case "("c
- Return MakeOpenParenToken(precedingTrivia, True)
-
- Case ")"c
- Return MakeCloseParenToken(precedingTrivia, True)
-
- Case "{"c
- Return MakeOpenBraceToken(precedingTrivia, True)
-
- Case "}"c
- Return MakeCloseBraceToken(precedingTrivia, True)
-
- Case ","c
- Return MakeCommaToken(precedingTrivia, True)
-
- Case "#"c
- Dim dl = ScanDateLiteral(precedingTrivia)
- If dl IsNot Nothing Then
- Return dl
- Else
- Return MakeHashToken(precedingTrivia, True)
- End If
-
- Case "&"c
- If CanGet(1) AndAlso BeginsBaseLiteral(Peek(1)) Then
- Return ScanNumericLiteral(precedingTrivia)
- End If
-
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeAmpersandEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakeAmpersandToken(precedingTrivia, True)
- End If
-
- Case "="c
- Return MakeEqualsToken(precedingTrivia, True)
-
- Case "<"c
- Return ScanLeftAngleBracket(precedingTrivia, True, _scanSingleLineTriviaFunc)
-
- Case ">"c
- Return ScanRightAngleBracket(precedingTrivia, True)
-
- Case ":"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeColonEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return ScanColonAsStatementTerminator(precedingTrivia, True)
- End If
-
- Case "+"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakePlusEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakePlusToken(precedingTrivia, True)
- End If
-
- Case "-"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeMinusEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakeMinusToken(precedingTrivia, True)
- End If
-
- Case "*"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeAsteriskEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakeAsteriskToken(precedingTrivia, True)
- End If
-
- Case "/"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakeSlashToken(precedingTrivia, True)
- End If
-
- Case "\"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeBackSlashEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakeBackslashToken(precedingTrivia, True)
- End If
-
- Case "^"c
- Dim lengthWithMaybeEquals = 1
- If TrySkipFollowingEquals(lengthWithMaybeEquals) Then
- Return MakeCaretEqualsToken(precedingTrivia, lengthWithMaybeEquals)
- Else
- Return MakeCaretToken(precedingTrivia, True)
- End If
-
- Case "!"c
- Return MakeExclamationToken(precedingTrivia, True)
-
- Case "."c
- If CanGet(1) AndAlso IsDecimalDigit(Peek(1)) Then
- Return ScanNumericLiteral(precedingTrivia)
- Else
- Return MakeDotToken(precedingTrivia, True)
- End If
-
- Case "0"c,
- "1"c,
- "2"c,
- "3"c,
- "4"c,
- "5"c,
- "6"c,
- "7"c,
- "8"c,
- "9"c
- Return ScanNumericLiteral(precedingTrivia)
-
- Case """"c
- Return ScanStringLiteral(precedingTrivia)
-
- Case "A"c
- If NextAre(1, "s ") Then
- Dim spelling = GetText(2)
- Return MakeKeyword(SyntaxKind.AsKeyword, spelling, precedingTrivia)
- Else
- Return ScanIdentifierOrKeyword(precedingTrivia)
- End If
-
- Case "E"c
- If NextAre(1, "nd ") Then
- Dim spelling = GetText(3)
- Return MakeKeyword(SyntaxKind.EndKeyword, spelling, precedingTrivia)
- Else
- Return ScanIdentifierOrKeyword(precedingTrivia)
- End If
-
- Case "I"c
- If NextAre(1, "f ") Then
- ' TODO: do we allow widechars in keywords?
- Dim spelling = GetText(2)
- Return MakeKeyword(SyntaxKind.IfKeyword, spelling, precedingTrivia)
- Else
- Return ScanIdentifierOrKeyword(precedingTrivia)
- End If
-
- Case "a"c, "b"c, "c"c, "d"c, "e"c, "f"c, "g"c, "h"c, "i"c, "j"c, "k"c, "l"c, "m"c,
- "n"c, "o"c, "p"c, "q"c, "r"c, "s"c, "t"c, "u"c, "v"c, "w"c, "x"c, "y"c, "z"c
- Return ScanIdentifierOrKeyword(precedingTrivia)
-
- Case "B"c, "C"c, "D"c, "F"c, "G"c, "H"c, "J"c, "K"c, "L"c, "M"c, "N"c, "O"c, "P"c, "Q"c,
- "R"c, "S"c, "T"c, "U"c, "V"c, "W"c, "X"c, "Y"c, "Z"c
- Return ScanIdentifierOrKeyword(precedingTrivia)
-
- Case "_"c
- If CanGet(1) AndAlso IsIdentifierPartCharacter(Peek(1)) Then
- Return ScanIdentifierOrKeyword(precedingTrivia)
- End If
-
- Dim err As ERRID = ERRID.ERR_ExpectedIdentifier
- Dim len = GetWhitespaceLength(1)
- If Not CanGet(len) OrElse IsNewLine(Peek(len)) OrElse PeekStartComment(len) > 0 Then
- err = ERRID.ERR_LineContWithCommentOrNoPrecSpace
- End If
-
- ' not a line continuation and cannot start identifier.
- Return MakeBadToken(precedingTrivia, 1, err)
-
- Case "["c
- Return ScanBracketedIdentifier(precedingTrivia)
-
- Case "?"c
- Return MakeQuestionToken(precedingTrivia, True)
-
- Case "%"c
- If CanGet(1) AndAlso
- Peek(1) = ">"c Then
- Return XmlMakeEndEmbeddedToken(precedingTrivia, _scanSingleLineTriviaFunc)
- End If
-
- End Select
-
- If IsIdentifierStartCharacter(ch) Then
- Return ScanIdentifierOrKeyword(precedingTrivia)
+ ' at this point it is very likely that we are located at
+ ' the beginning of a token
+ Private Function TryScanToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
+ Dim ch As Char
+ If Not Peep(ch) Then
+ Return MakeEofToken(precedingTrivia)
End If
+ Return ScanTokenCommon(precedingTrivia, ch, False)
+ End Function
- Debug.Assert(Not IsNewLine(ch))
- Debug.Assert(Not IsDoubleQuote(ch))
-
- Return Nothing
+ Private Function ScanTokenFullWidth(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), ch As Char) As SyntaxToken
+ Return ScanTokenCommon(precedingTrivia, ch, True)
End Function
' // Allow whitespace between the characters of a two-character token.
@@ -1449,11 +1170,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim Here = Index
Dim eq As Char
- While CanGet(Here)
- eq = Peek(Here)
+ While Peep(Here, eq)
Here += 1
If Not IsWhitespace(eq) Then
- If eq = "="c OrElse eq = FULLWIDTH_EQUALS_SIGN Then
+ If eq.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then
Index = Here
Return True
Else
@@ -1465,21 +1185,23 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function ScanRightAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean) As SyntaxToken
- Debug.Assert(CanGet) ' >
- Debug.Assert(Peek() = ">"c OrElse Peek() = FULLWIDTH_GREATER_THAN_SIGN)
+ Dim c As Char
+#If DEBUG Then
+ Dim ok = Peep(c)
+ Debug.Assert(ok) ' >
+ Debug.Assert(c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN))
+#End If
Dim length As Integer = 1
' // Allow whitespace between the characters of a two-character token.
length = GetWhitespaceLength(length)
- If CanGet(length) Then
- Dim c As Char = Peek(length)
-
- If c = "="c OrElse c = FULLWIDTH_EQUALS_SIGN Then
+ If Peep(length, c) Then
+ If c.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then
length += 1
Return MakeGreaterThanEqualsToken(precedingTrivia, length)
- ElseIf c = ">"c OrElse c = FULLWIDTH_GREATER_THAN_SIGN Then
+ ElseIf c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then
length += 1
If TrySkipFollowingEquals(length) Then
Return MakeGreaterThanGreaterThanEqualsToken(precedingTrivia, length)
@@ -1492,33 +1214,31 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function ScanLeftAngleBracket(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), charIsFullWidth As Boolean, scanTrailingTrivia As ScanTriviaFunc) As SyntaxToken
- Debug.Assert(CanGet) ' <
- Debug.Assert(Peek() = "<"c OrElse Peek() = FULLWIDTH_LESS_THAN_SIGN)
-
+ Dim c As Char
+#If DEBUG Then
+ Dim ok = Peep(c)
+ Debug.Assert(ok) ' <
+ Debug.Assert(c.IsFrom("<"c, FULLWIDTH_LESS_THAN_SIGN))
+#End If
Dim length As Integer = 1
-
' Check for XML tokens
- If Not charIsFullWidth AndAlso CanGet(length) Then
- Dim c As Char = Peek(length)
+ If Not charIsFullWidth AndAlso Peep(length, c) Then
Select Case c
Case "!"c
- If CanGet(length + 2) Then
+ If Peep(length + 2, c) Then
Select Case (Peek(length + 1))
Case "-"c
- If CanGet(length + 3) AndAlso Peek(length + 2) = "-"c Then
+ If CanGet(length + 3) AndAlso c = "-"c Then
Return XmlMakeBeginCommentToken(precedingTrivia, scanTrailingTrivia)
End If
Case "["c
-
If NextAre(length + 2, "CDATA[") Then
-
Return XmlMakeBeginCDataToken(precedingTrivia, scanTrailingTrivia)
End If
End Select
End If
Case "?"c
Return XmlMakeBeginProcessingInstructionToken(precedingTrivia, scanTrailingTrivia)
-
Case "/"c
Return XmlMakeBeginEndElementToken(precedingTrivia, _scanSingleLineTriviaFunc)
End Select
@@ -1526,24 +1246,18 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' // Allow whitespace between the characters of a two-character token.
length = GetWhitespaceLength(length)
-
- If CanGet(length) Then
- Dim c As Char = Peek(length)
-
- If c = "="c OrElse c = FULLWIDTH_EQUALS_SIGN Then
+ If Peep(length, c) Then
+ If c.IsFrom("="c, FULLWIDTH_EQUALS_SIGN) Then
length += 1
Return MakeLessThanEqualsToken(precedingTrivia, length)
- ElseIf c = ">"c OrElse c = FULLWIDTH_GREATER_THAN_SIGN Then
+ ElseIf c.IsFrom(">"c, FULLWIDTH_GREATER_THAN_SIGN) Then
length += 1
Return MakeLessThanGreaterThanToken(precedingTrivia, length)
- ElseIf c = "<"c OrElse c = FULLWIDTH_LESS_THAN_SIGN Then
+ ElseIf c.IsFrom("<"c, FULLWIDTH_LESS_THAN_SIGN) Then
length += 1
-
- If CanGet(length) Then
- c = Peek(length)
-
+ If Peep(length, c) Then
'if the second "<" is a part of "<%" - like in "<<%" , we do not want to use it.
- If c <> "%"c AndAlso c <> FULLWIDTH_PERCENT_SIGN Then
+ If c.IsnotFrom("%"c, FULLWIDTH_PERCENT_SIGN) Then
If TrySkipFollowingEquals(length) Then
Return MakeLessThanLessThanEqualsToken(precedingTrivia, length)
Else
@@ -1568,10 +1282,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim c = spelling(0)
If SyntaxFacts.IsIdentifierStartCharacter(c) Then
- ' SPEC: ... Visual Basic identifiers conform to the Unicode Standard Annex 15 with one
- ' SPEC: exception: identifiers may begin with an underscore (connector) character.
- ' SPEC: If an identifier begins with an underscore, it must contain at least one other
- ' SPEC: valid identifier character to disambiguate it from a line continuation.
+ ' SPEC: ... Visual Basic identifiers conform to the Unicode Standard Annex 15 with one
+ ' SPEC: exception: identifiers may begin with an underscore (connector) character.
+ ' SPEC: If an identifier begins with an underscore, it must contain at least one other
+ ' SPEC: valid identifier character to disambiguate it from a line continuation.
If IsConnectorPunctuation(c) AndAlso spellingLength = 1 Then
Return False
End If
@@ -1587,14 +1301,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function ScanIdentifierOrKeyword(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
- Debug.Assert(CanGet)
- Debug.Assert(IsIdentifierStartCharacter(Peek))
- Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller
+ Dim ch, cx As Char
+ Dim ok = Peep(ch) ' Non DEBUG usage see:= If IsConnectorPunctuation(ch)
- Dim ch = Peek()
- If CanGet(1) Then
- Dim ch1 = Peek(1)
- If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(ch1) Then
+ Debug.Assert(ok)
+ Debug.Assert(IsIdentifierStartCharacter(ch))
+ Debug.Assert(PeekStartComment(0) = 0) ' comment should be handled by caller
+ If Peep(1, cx) Then
+ If IsConnectorPunctuation(ch) AndAlso Not IsIdentifierPartCharacter(cx) Then
Return MakeBadToken(precedingTrivia, 1, ERRID.ERR_ExpectedIdentifier)
End If
End If
@@ -1604,8 +1318,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' // The C++ compiler refuses to inline IsIdentifierCharacter, so the
' // < 128 test is inline here. (This loop gets a *lot* of traffic.)
' TODO: make sure we get good perf here
- While CanGet(len)
- ch = Peek(len)
+ While Peep(len, ch)
Dim code = Convert.ToUInt16(ch)
If code < 128 AndAlso IsNarrowIdentifierCharacter(code) OrElse
@@ -1619,18 +1332,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'Check for a type character
Dim TypeCharacter As TypeCharacter = TypeCharacter.None
- If CanGet(len) Then
- ch = Peek(len)
+ If Peep(len, ch) Then
FullWidthRepeat:
Select Case ch
Case "!"c
' // If the ! is followed by an identifier it is a dictionary lookup operator, not a type character.
- If CanGet(len + 1) Then
- Dim NextChar As Char = Peek(len + 1)
-
- If IsIdentifierStartCharacter(NextChar) OrElse
- MatchOneOrAnotherOrFullwidth(NextChar, "["c, "]"c) Then
+ If Peep(len + 1, cx) Then
+ If IsIdentifierStartCharacter(cx) OrElse
+ MatchOneOrAnotherOrFullwidth(cx, "["c, "]"c) Then
Exit Select
End If
End If
@@ -1669,9 +1379,7 @@ FullWidthRepeat:
Dim contextualKind As SyntaxKind = SyntaxKind.IdentifierToken
Dim spelling = GetText(len)
- Dim BaseSpelling = If(TypeCharacter = TypeCharacter.None,
- spelling,
- Intern(spelling, 0, len - 1))
+ Dim BaseSpelling = If(TypeCharacter = TypeCharacter.None, spelling, Intern(spelling, 0, len - 1))
' this can be keyword only if it has no type character, or if it is Mid$
If TypeCharacter = TypeCharacter.None Then
@@ -1705,34 +1413,36 @@ FullWidthRepeat:
End Function
Private Function ScanBracketedIdentifier(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
- Debug.Assert(CanGet) ' [
- Debug.Assert(Peek() = "["c OrElse Peek() = FULLWIDTH_LEFT_SQUARE_BRACKET)
+ Dim ch As Char
+#If DEBUG Then
+ Dim ok = Peep(ch)
+ Debug.Assert(ok) ' [
+ Debug.Assert(ch.IsFrom("["c, FULLWIDTH_LEFT_SQUARE_BRACKET))
+#End If
Dim IdStart As Integer = 1
Dim Here As Integer = IdStart
Dim InvalidIdentifier As Boolean = False
- If Not CanGet(Here) Then
+ If Not Peep(Here, ch) Then
Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_MissingEndBrack)
End If
- Dim ch = Peek(Here)
+ Dim cx As Char
' check if we can start an ident.
If Not IsIdentifierStartCharacter(ch) OrElse
(IsConnectorPunctuation(ch) AndAlso
- Not (CanGet(Here + 1) AndAlso
- IsIdentifierPartCharacter(Peek(Here + 1)))) Then
+ Not (Peep(Here + 1, cx) AndAlso IsIdentifierPartCharacter(cx))) Then
InvalidIdentifier = True
End If
' check ident until ]
- While CanGet(Here)
- Dim [Next] As Char = Peek(Here)
+ While Peep(Here, cx)
- If [Next] = "]"c OrElse [Next] = FULLWIDTH_RIGHT_SQUARE_BRACKET Then
+ If cx.IsFrom("]"c, FULLWIDTH_RIGHT_SQUARE_BRACKET) Then
Dim IdStringLength As Integer = Here - IdStart
If IdStringLength > 0 AndAlso Not InvalidIdentifier Then
@@ -1754,9 +1464,9 @@ FullWidthRepeat:
' // The sequence "[]" does not define a valid identifier.
Return MakeBadToken(precedingTrivia, Here + 1, ERRID.ERR_ExpectedIdentifier)
End If
- ElseIf IsNewLine([Next]) Then
+ ElseIf IsNewLine(cx) Then
Exit While
- ElseIf Not IsIdentifierPartCharacter([Next]) Then
+ ElseIf Not IsIdentifierPartCharacter(cx) Then
InvalidIdentifier = True
Exit While
End If
@@ -1778,7 +1488,9 @@ FullWidthRepeat:
End Enum
Private Function ScanNumericLiteral(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As SyntaxToken
- Debug.Assert(CanGet)
+ Dim ch As Char
+ Dim ok = Peep(ch)
+ Debug.Assert(ok)
Dim Here As Integer = 0
Dim IntegerLiteralStart As Integer
@@ -1792,10 +1504,9 @@ FullWidthRepeat:
' // First read a leading base specifier, if present, followed by a sequence of zero
' // or more digits.
- Dim ch = Peek()
- If ch = "&"c OrElse ch = FULLWIDTH_AMPERSAND Then
+ If ch.IsFrom("&"c, FULLWIDTH_AMPERSAND) Then
Here += 1
- ch = If(CanGet(Here), Peek(Here), ChrW(0))
+ If Not Peep(Here, ch) Then ch = ChrW(0)
FullWidthRepeat:
Select Case ch
@@ -1804,11 +1515,7 @@ FullWidthRepeat:
IntegerLiteralStart = Here
Base = LiteralBase.Hexadecimal
- While CanGet(Here)
- ch = Peek(Here)
- If Not IsHexDigit(ch) Then
- Exit While
- End If
+ While Peep(Here, ch) AndAlso IsHexDigit(ch)
Here += 1
End While
@@ -1817,11 +1524,7 @@ FullWidthRepeat:
IntegerLiteralStart = Here
Base = LiteralBase.Octal
- While CanGet(Here)
- ch = Peek(Here)
- If Not IsOctalDigit(ch) Then
- Exit While
- End If
+ While Peep(Here, ch) AndAlso IsOctalDigit(ch)
Here += 1
End While
@@ -1836,11 +1539,7 @@ FullWidthRepeat:
Else
' no base specifier - just go through decimal digits.
IntegerLiteralStart = Here
- While CanGet(Here)
- ch = Peek(Here)
- If Not IsDecimalDigit(ch) Then
- Exit While
- End If
+ While Peep(Here, ch) AndAlso IsDecimalDigit(ch)
Here += 1
End While
End If
@@ -1850,21 +1549,16 @@ FullWidthRepeat:
' // Unless there was an explicit base specifier (which indicates an integer literal),
' // read the rest of a float literal.
- If Base = LiteralBase.Decimal AndAlso CanGet(Here) Then
+ If Base = LiteralBase.Decimal AndAlso Peep(Here, ch) Then
' // First read a '.' followed by a sequence of one or more digits.
- ch = Peek(Here)
- If (ch = "."c Or ch = FULLWIDTH_FULL_STOP) AndAlso
- CanGet(Here + 1) AndAlso
- IsDecimalDigit(Peek(Here + 1)) Then
+ Dim cx As Char
+ If ch.IsFrom("."c, FULLWIDTH_FULL_STOP) AndAlso
+ (Peep(Here + 1, cx) AndAlso IsDecimalDigit(cx)) Then
Here += 2 ' skip dot and first digit
' all following decimal digits belong to the literal (fractional part)
- While CanGet(Here)
- ch = Peek(Here)
- If Not IsDecimalDigit(ch) Then
- Exit While
- End If
+ While Peep(Here, ch) AndAlso IsDecimalDigit(ch)
Here += 1
End While
literalKind = NumericLiteralKind.Float
@@ -1872,24 +1566,16 @@ FullWidthRepeat:
' // Read an exponent symbol followed by an optional sign and a sequence of
' // one or more digits.
- If CanGet(Here) AndAlso BeginsExponent(Peek(Here)) Then
+ If Peep(Here, cx) AndAlso BeginsExponent(cx) Then
Here += 1
- If CanGet(Here) Then
- ch = Peek(Here)
-
- If MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then
- Here += 1
- End If
+ If Peep(Here, ch) AndAlso MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then
+ Here += 1
End If
- If CanGet(Here) AndAlso IsDecimalDigit(Peek(Here)) Then
+ If Peep(Here, cx) AndAlso IsDecimalDigit(cx) Then
Here += 1
- While CanGet(Here)
- ch = Peek(Here)
- If Not IsDecimalDigit(ch) Then
- Exit While
- End If
+ While Peep(Here, ch) AndAlso IsDecimalDigit(ch)
Here += 1
End While
Else
@@ -1908,8 +1594,7 @@ FullWidthRepeat:
Dim TypeCharacter As TypeCharacter = TypeCharacter.None
- If CanGet(Here) Then
- ch = Peek(Here)
+ If Peep(Here, ch) Then
FullWidthRepeat2:
Select Case ch
@@ -1985,9 +1670,7 @@ FullWidthRepeat2:
literalKind = NumericLiteralKind.Decimal
' check if this was not attempt to use obsolete exponent
- If CanGet(Here + 1) Then
- ch = Peek(Here + 1)
-
+ If Peep(Here + 1, ch) Then
If IsDecimalDigit(ch) OrElse MatchOneOrAnotherOrFullwidth(ch, "+"c, "-"c) Then
Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_ObsoleteExponent)
End If
@@ -1997,9 +1680,8 @@ FullWidthRepeat2:
End If
Case "U"c, "u"c
- If literalKind <> NumericLiteralKind.Float AndAlso CanGet(Here + 1) Then
- Dim NextChar As Char = Peek(Here + 1)
-
+ Dim NextChar As Char
+ If literalKind <> NumericLiteralKind.Float AndAlso Peep(Here + 1, NextChar) Then
'unsigned suffixes - US, UL, UI
If MatchOneOrAnotherOrFullwidth(NextChar, "S"c, "s"c) Then
TypeCharacter = TypeCharacter.UShortLiteral
@@ -2103,7 +1785,7 @@ FullWidthRepeat2:
End If
Else
- ' // Copy the text of the literal to deal with fullwidth
+ ' // Copy the text of the literal to deal with fullwidth
Dim scratch = GetScratch()
For i = 0 To literalWithoutTypeChar - 1
Dim curCh = Peek(i)
@@ -2170,31 +1852,17 @@ FullWidthRepeat2:
Return Decimal.TryParse(text, NumberStyles.AllowDecimalPoint Or NumberStyles.AllowExponent, CultureInfo.InvariantCulture, value)
End Function
- Private Function ScanIntLiteral(
- ByRef ReturnValue As Integer,
- ByRef Here As Integer
- ) As Boolean
+ Private Function ScanIntLiteral(ByRef ReturnValue As Integer, ByRef Here As Integer) As Boolean
Debug.Assert(Here >= 0)
-
- If Not CanGet(Here) Then
- Return False
- End If
-
- Dim ch = Peek(Here)
- If Not IsDecimalDigit(ch) Then
+ Dim ch As Char
+ If Not Peep(Here, ch) OrElse Not IsDecimalDigit(ch) Then
Return False
End If
Dim IntegralValue As Integer = IntegralLiteralCharacterValue(ch)
Here += 1
- While CanGet(Here)
- ch = Peek(Here)
-
- If Not IsDecimalDigit(ch) Then
- Exit While
- End If
-
+ While Peep(Here, ch) AndAlso IsDecimalDigit(ch)
Dim nextDigit = IntegralLiteralCharacterValue(ch)
If IntegralValue < 214748364 OrElse
(IntegralValue = 214748364 AndAlso nextDigit < 8) Then
@@ -2239,19 +1907,19 @@ FullWidthRepeat2:
' // The first thing has to be an integer, although it's not clear what it is yet
If Not ScanIntLiteral(FirstValue, Here) Then
Return Nothing
-
End If
' // If we see a /, then it's a date
+ Dim c As Char
- If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then
+ If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then
Dim FirstDateSeparator As Integer = Here
' // We've got a date
HaveDateValue = True
Here += 1
- ' Is the first value a year?
+ ' Is the first value a year?
' It is a year if it consists of exactly 4 digits.
' Condition below uses 5 because we already skipped the separator.
If Here - FirstValueStart = 5 Then
@@ -2264,10 +1932,10 @@ FullWidthRepeat2:
End If
' Do we have a day value?
- If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then
+ If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then
' // Check to see they used a consistent separator
- If Peek(Here) <> Peek(FirstDateSeparator) Then
+ If c <> Peek(FirstDateSeparator) Then
GoTo baddate
End If
@@ -2289,11 +1957,10 @@ FullWidthRepeat2:
End If
' // Do we have a year value?
-
- If CanGet(Here) AndAlso IsDateSeparatorCharacter(Peek(Here)) Then
+ If Peep(Here, c) AndAlso IsDateSeparatorCharacter(c) Then
' // Check to see they used a consistent separator
- If Peek(Here) <> Peek(FirstDateSeparator) Then
+ If c <> Peek(FirstDateSeparator) Then
GoTo baddate
End If
@@ -2332,8 +1999,7 @@ FullWidthRepeat2:
If HaveTimeValue Then
' // Do we see a :?
-
- If CanGet(Here) AndAlso IsColon(Peek(Here)) Then
+ If Peep(Here, c) AndAlso IsColon(c) Then
Here += 1
' // Now let's get the minute value
@@ -2345,8 +2011,7 @@ FullWidthRepeat2:
HaveMinuteValue = True
' // Do we have a second value?
-
- If CanGet(Here) AndAlso IsColon(Peek(Here)) Then
+ If Peep(Here, c) AndAlso IsColon(c) Then
' // Yes.
HaveSecondValue = True
Here += 1
@@ -2361,24 +2026,21 @@ FullWidthRepeat2:
' // Check AM/PM
- If CanGet(Here) Then
- If Peek(Here) = "A"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_A OrElse
- Peek(Here) = "a"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_A Then
-
+ If Peep(Here, c) Then
+ If c.IsFrom("A"c, FULLWIDTH_LATIN_CAPITAL_LETTER_A,
+ "a"c, FULLWIDTH_LATIN_SMALL_LETTER_A) Then
HaveAM = True
Here += 1
- ElseIf Peek(Here) = "P"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_P OrElse
- Peek(Here) = "p"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_P Then
-
+ ElseIf c.IsFrom("P"c, FULLWIDTH_LATIN_CAPITAL_LETTER_P,
+ "p"c, FULLWIDTH_LATIN_SMALL_LETTER_P) Then
HavePM = True
Here += 1
End If
-
- If CanGet(Here) AndAlso (HaveAM OrElse HavePM) Then
- If Peek(Here) = "M"c OrElse Peek(Here) = FULLWIDTH_LATIN_CAPITAL_LETTER_M OrElse
- Peek(Here) = "m"c OrElse Peek(Here) = FULLWIDTH_LATIN_SMALL_LETTER_M Then
+ If Peep(Here, c) AndAlso (HaveAM OrElse HavePM) Then
+ If c.IsFrom("M"c, FULLWIDTH_LATIN_CAPITAL_LETTER_M,
+ "m"c, FULLWIDTH_LATIN_SMALL_LETTER_M) Then
Here = GetWhitespaceLength(Here + 1)
@@ -2395,7 +2057,7 @@ FullWidthRepeat2:
End If
End If
- If Not CanGet(Here) OrElse Not IsHash(Peek(Here)) Then
+ If Not Peep(Here, c) OrElse Not IsHash(c) Then
GoTo baddate
End If
@@ -2500,16 +2162,11 @@ FullWidthRepeat2:
baddate:
' // If we can find a closing #, then assume it's a malformed date,
' // otherwise, it's not a date
-
- While CanGet(Here)
- Dim ch As Char = Peek(Here)
- If IsHash(ch) OrElse IsNewLine(ch) Then
- Exit While
- End If
+ While Peep(Here, c) AndAlso Not (IsHash(c) OrElse IsNewLine(c))
Here += 1
End While
- If Not CanGet(Here) OrElse IsNewLine(Peek(Here)) Then
+ If Not Peep(Here, c) OrElse IsNewLine(c) Then
' // No closing #
Return Nothing
Else
@@ -2530,38 +2187,29 @@ baddate:
' // Check for a Char literal, which can be of the form:
' // """"c or ""c
- If CanGet(3) AndAlso IsDoubleQuote(Peek(2)) Then
+ If Peep(3, ch) AndAlso IsDoubleQuote(Peek(2)) Then
If IsDoubleQuote(Peek(1)) Then
- If IsDoubleQuote(Peek(3)) AndAlso
- CanGet(4) AndAlso
- IsLetterC(Peek(4)) Then
-
+ If IsDoubleQuote(ch) AndAlso Peep(4, ch) AndAlso IsLetterC(ch) Then
' // Double-quote Char literal: """"c
Return MakeCharacterLiteralToken(precedingTrivia, """"c, 5)
End If
- ElseIf IsLetterC(Peek(3)) Then
+ ElseIf IsLetterC(ch) Then
' // Char literal. "x"c
Return MakeCharacterLiteralToken(precedingTrivia, Peek(1), 4)
End If
End If
- If CanGet(2) AndAlso
- IsDoubleQuote(Peek(1)) AndAlso
- IsLetterC(Peek(2)) Then
-
+ If Peep(2, ch) AndAlso IsDoubleQuote(Peek(1)) AndAlso IsLetterC(ch) Then
' // Error. ""c is not a legal char constant
Return MakeBadToken(precedingTrivia, 3, ERRID.ERR_IllegalCharConstant)
End If
Dim scratch = GetScratch()
- While CanGet(length)
- ch = Peek(length)
+ While Peep(length, ch)
If IsDoubleQuote(ch) Then
- If CanGet(length + 1) Then
- ch = Peek(length + 1)
-
+ If Peep(length + 1, ch) Then
If IsDoubleQuote(ch) Then
' // An escaped double quote
scratch.Append(""""c)
@@ -2571,7 +2219,6 @@ baddate:
' // The end of the char literal.
If IsLetterC(ch) Then
' // Error. "aad"c is not a legal char constant
-
' // +2 to include both " and c in the token span
scratch.Clear()
Return MakeBadToken(precedingTrivia, length + 2, ERRID.ERR_IllegalCharConstant)
@@ -2598,7 +2245,7 @@ baddate:
' CC has trouble to prove this after the loop
Debug.Assert(CanGet(length - 1))
- '// The literal does not have an explicit termination.
+ '// The literal does not have an explicit termination.
' DIFFERENT: here in IDE we used to report string token marked as unterminated
Dim sp = GetTextNotInterned(length)
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb
index 30e698a182c37..5a8ae75621b38 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerBuffer.vb
@@ -1,9 +1,10 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
+Imports System.Runtime.InteropServices
Imports System.Text
Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.VisualBasic.Symbols
@@ -95,6 +96,57 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return p
End Function
+ '''
+ ''' Peep at the index position, if it is reachable pass the char back
+ '''
+ ''' How far in front do what to try "peeking" at?
+ ''' (Passback) the character at that location if possible.
+ ''' If not the contents are left unaffected.
+ ''' NOTE: This is different behaviour to the Try methods that always return a value.
+ '''
+ ''' Returns True if it possible to peek at, otherwise returns False.
+ Friend Function Peep(at As Integer, ByRef ch As Char) As Boolean
+ ' CanGet(at)
+ Debug.Assert(_lineBufferOffset + at >= 0)
+ Debug.Assert(at >= -MaxCharsLookBehind)
+ If _lineBufferOffset + at >= _bufferLen Then
+ Return False
+ End If
+ ' Peek(at)
+ at += _lineBufferOffset
+ Dim page = _curPage
+ ch = page._arr(at And s_PAGE_MASK)
+ If page._pageStart <> (at And s_NOT_PAGE_MASK) Then
+ page = GetPage(at)
+ ch = page._arr(at And s_PAGE_MASK)
+ End If
+ Return True
+ End Function
+
+ '''
+ ''' Peep at the next index position, if it is reachable pass the char back
+ '''
+ ''' (Passback) the character at that location if possible.
+ ''' If not the contents are left unaffected.
+ ''' NOTE: This is different behaviour to the Try methods that always return a value.
+ '''
+ ''' Returns True if it possible to peek at, otherwise returns False.
+ Friend Function Peep(ByRef ch As Char) As Boolean
+ ' CanGet()
+ If _lineBufferOffset >= _bufferLen Then
+ Return False
+ End If
+ ' Peek()
+ Dim page = _curPage
+ Dim position = _lineBufferOffset
+ ch = page._arr(position And s_PAGE_MASK)
+ If page._pageStart <> (position And s_NOT_PAGE_MASK) Then
+ page = GetPage(position)
+ ch = page._arr(position And s_PAGE_MASK)
+ End If
+ Return True
+ End Function
+
' PERF CRITICAL
Private Function Peek(skip As Integer) As Char
Debug.Assert(CanGet(skip))
@@ -214,5 +266,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End If
Return result
End Function
+
End Class
+
End Namespace
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb
index 40a522c7dbfdd..075c1e4ad674e 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerInterpolatedString.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Imports Microsoft.CodeAnalysis.VisualBasic.SyntaxFacts
@@ -19,15 +19,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim leadingTriviaLength = GetWhitespaceLength(0)
Dim offset = leadingTriviaLength
Dim length As Integer
-
- If Not CanGet(offset) Then
+ Dim c As Char
+ If Not Peep(offset, c) Then
Return MakeEndOfInterpolatedStringToken()
End If
- Dim c = Peek(offset)
-
' This should only ever happen for $" or }
- Debug.Assert(leadingTriviaLength = 0 OrElse c = "$"c OrElse c = FULLWIDTH_DOLLAR_SIGN OrElse IsRightCurlyBracket(c))
+ Debug.Assert(leadingTriviaLength = 0 OrElse c.IsFrom("$"c, FULLWIDTH_DOLLAR_SIGN) OrElse IsRightCurlyBracket(c))
' Another } may follow the close brace of an interpolation if the interpolation lacked a format clause.
' This is because the normal escaping rules only apply when parsing the format string.
@@ -38,7 +36,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Select Case c
Case "$"c, FULLWIDTH_DOLLAR_SIGN
- If CanGet(offset + 1) AndAlso IsDoubleQuote(Peek(offset + 1)) Then
+ If Peep(offset + 1, c) AndAlso IsDoubleQuote(c) Then
kind = SyntaxKind.DollarSignDoubleQuoteToken
length = 2
scanTrailingTrivia = False ' Trailing whitespace should be scanned as interpolated string text.
@@ -112,22 +110,20 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function IsInterpolatedStringPunctuation(Optional offset As Integer = 0) As Boolean
- If Not CanGet(offset) Then Return False
-
- Dim c = Peek(offset)
-
+ Dim c As Char
+ If Not Peep(offset, c) Then Return False
If IsLeftCurlyBracket(c) Then
- Return Not CanGet(offset + 1) OrElse Not IsLeftCurlyBracket(Peek(offset + 1))
+ Return Not Peep(offset + 1, c) OrElse Not IsLeftCurlyBracket(c)
ElseIf IsRightCurlyBracket(c) Then
- Return Not CanGet(offset + 1) OrElse Not IsRightCurlyBracket(Peek(offset + 1))
+ Return Not Peep(offset + 1, c) OrElse Not IsRightCurlyBracket(c)
ElseIf IsDoubleQuote(c)
'A subtle difference between this case and the one above.
' In both interpolated and literal strings the two quote characters used in an escape sequence don't have to match.
' It's enough that the next character is *a* quote char. It doesn't have to be the same quote.
' If we want to preserve consistency the quotes need to be special cased.
- Return Not CanGet(offset + 1) OrElse Not IsDoubleQuote(Peek(offset + 1))
+ Return Not Peep(offset + 1, c) OrElse Not IsDoubleQuote(c)
Else
Return False
@@ -140,17 +136,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim offset = 0
Dim pendingWhitespace = 0
Dim valueBuilder = GetScratch()
-
- Do While CanGet(offset)
-
- Dim c = Peek(offset)
+ Dim c As Char
+ Do While Peep(offset, c)
' Any combination of fullwidth and ASCII curly braces of the same direction is an escaping sequence for the corresponding ASCII curly brace.
' We insert that curly brace doubled and because this is the escaping sequence understood by String.Format, that will be replaced by a single brace.
' This is deliberate design and it aligns with existing rules for double quote escaping in strings.
If IsLeftCurlyBracket(c) Then
-
- If CanGet(offset + 1) AndAlso IsLeftCurlyBracket(Peek(offset + 1)) Then
+ If Peep(offset + 1, c) AndAlso IsLeftCurlyBracket(c) Then
' This is an escape sequence.
valueBuilder.Append("{{")
@@ -163,8 +156,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Exit Do
ElseIf IsRightCurlyBracket(c) Then
-
- If CanGet(offset + 1) AndAlso IsRightCurlyBracket(Peek(offset + 1)) Then
+ If Peep(offset + 1, c) AndAlso IsRightCurlyBracket(c) Then
' This is an escape sequence.
valueBuilder.Append("}}")
@@ -177,8 +169,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Exit Do
ElseIf IsDoubleQuote(c)
-
- If CanGet(offset + 1) AndAlso IsDoubleQuote(Peek(offset + 1)) Then
+ If Peep(offset + 1, c) AndAlso IsDoubleQuote(c) Then
' This is a VB double quote escape. Oddly enough this logic allows mixing and matching of
' smart and dumb double quotes in any order. Regardless we always emit as a standard double quote.
' This is consistent with their handling in string literals.
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb
index 5e6e421df0beb..a00782ff946f3 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/ScannerXml.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Compare Binary
Option Strict On
@@ -16,15 +16,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function ScanXmlTrivia(c As Char) As SyntaxList(Of VisualBasicSyntaxNode)
Debug.Assert(Not IsScanningXmlDoc)
- Debug.Assert(c = CARRIAGE_RETURN OrElse c = LINE_FEED OrElse c = " "c OrElse c = CHARACTER_TABULATION)
+ Debug.Assert(c.IsFrom(CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION))
Dim builder = _triviaListPool.Allocate
Dim len = 0
Do
- If c = " "c OrElse c = CHARACTER_TABULATION Then
+ If c.IsFrom(" "c, CHARACTER_TABULATION) Then
len += 1
- ElseIf c = CARRIAGE_RETURN OrElse c = LINE_FEED Then
+ ElseIf c.IsFrom(CARRIAGE_RETURN, LINE_FEED) Then
If len > 0 Then
builder.Add(MakeWhiteSpaceTrivia(GetText(len)))
len = 0
@@ -34,10 +34,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Exit Do
End If
- If Not CanGet(len) Then
+ If Not Peep(len, c) Then
Exit Do
End If
- c = Peek(len)
Loop
If len > 0 Then
@@ -69,8 +68,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim leadingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing
- While CanGet()
- Dim c As Char = Peek()
+ Dim c As Char
+ While Peep(c)
Select Case (c)
' // Whitespace
@@ -93,7 +92,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
leadingTrivia = ScanXmlTrivia(c)
Case "/"c
- If CanGet(1) AndAlso Peek(1) = ">" Then
+ If NextIs(1, ">"c) Then
Return XmlMakeEndEmptyElementToken(leadingTrivia)
End If
Return XmlMakeDivToken(leadingTrivia)
@@ -113,29 +112,30 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return XmlMakeDoubleQuoteToken(leadingTrivia, c, isOpening:=True)
Case "<"c
- If CanGet(1) Then
- Dim ch As Char = Peek(1)
+ Dim ch As Char
+ If Peep(1, ch) Then
+ Dim c2 As Char
Select Case ch
Case "!"c
- If CanGet(2) Then
- Select Case (Peek(2))
+ If Peep(2, c2) Then
+ Select Case c2
Case "-"c
- If NextIs(3,"-"c) Then
+ If NextIs(3, "-"c) Then
Return XmlMakeBeginCommentToken(leadingTrivia, s_scanNoTriviaFunc)
End If
Case "["c
- If NextAre(3,"CDATA[") Then
+ If NextAre(3, "CDATA[") Then
Return XmlMakeBeginCDataToken(leadingTrivia, s_scanNoTriviaFunc)
End If
Case "D"c
- If Nextare(3,"OCTYPE") Then
+ If NextAre(3, "OCTYPE") Then
Return XmlMakeBeginDTDToken(leadingTrivia)
End If
End Select
End If
Return XmlLessThanExclamationToken(state, leadingTrivia)
Case "%"c
- If NextIs(2,"="c) Then
+ If NextIs(2, "="c) Then
Return XmlMakeBeginEmbeddedToken(leadingTrivia)
End If
Case "?"c
@@ -148,37 +148,36 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return XmlMakeLessToken(leadingTrivia)
Case "?"c
-
- If NextIs(1,">"c) Then
+ If NextIs(1, ">"c) Then
' // Create token for the '?>' termination sequence
Return XmlMakeEndProcessingInstructionToken(leadingTrivia)
- End If
+ End If
- Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar)
+ Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar)
- Case "("c
- Return XmlMakeLeftParenToken(leadingTrivia)
+ Case "("c
+ Return XmlMakeLeftParenToken(leadingTrivia)
- Case ")"c
- Return XmlMakeRightParenToken(leadingTrivia)
+ Case ")"c
+ Return XmlMakeRightParenToken(leadingTrivia)
- Case "!"c, ";"c, "#"c, ","c, "}"c
- Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar)
+ Case "!"c, ";"c, "#"c, ","c, "}"c
+ Return XmlMakeBadToken(leadingTrivia, 1, ERRID.ERR_IllegalXmlNameChar)
- Case ":"c
- Return XmlMakeColonToken(leadingTrivia)
+ Case ":"c
+ Return XmlMakeColonToken(leadingTrivia)
- Case "["c
- Return XmlMakeOpenBracketToken(state, leadingTrivia)
+ Case "["c
+ Return XmlMakeOpenBracketToken(state, leadingTrivia)
- Case "]"c
- Return XmlMakeCloseBracketToken(state, leadingTrivia)
+ Case "]"c
+ Return XmlMakeCloseBracketToken(state, leadingTrivia)
- Case Else
- ' // Because of weak scanning of QName, this state must always handle
- ' // '=' | '\'' | '"'| '/' | '>' | '<' | '?'
+ Case Else
+ ' // Because of weak scanning of QName, this state must always handle
+ ' // '=' | '\'' | '"'| '/' | '>' | '<' | '?'
- Return ScanXmlNcName(leadingTrivia)
+ Return ScanXmlNcName(leadingTrivia)
End Select
End While
@@ -186,7 +185,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
'//
- '// This is used to detect a VB statement on the next line
+ '// This is used to detect a VB statement on the next line
'//
'// NL WS* KW WS* ID | KW
'// Example Dim x
@@ -200,21 +199,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'//
'// NL WS* < ID WS* (
'// Example SyntaxKind.XmlNameToken Then
leadingTrivia = ScanSingleLineTrivia()
c = Peek()
- possibleStatement =
- c = "("c OrElse c = FULLWIDTH_LEFT_PARENTHESIS
+ possibleStatement = c.IsFrom("("c, FULLWIDTH_LEFT_PARENTHESIS)
End If
End If
@@ -309,8 +307,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim scratch = GetScratch()
- While CanGet(Here)
- Dim c As Char = Peek(Here)
+ Dim c As Char
+ While Peep(Here, c)
Select Case (c)
Case CARRIAGE_RETURN, LINE_FEED
@@ -336,28 +334,29 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return XmlMakeTextLiteralToken(Nothing, Here, scratch)
Else
scratch.Clear() ' will not use this
- Here = 0 ' consumed chars.
+ Here = 0 ' consumed chars.
precedingTrivia = ScanXmlTrivia(Peek)
End If
End If
Debug.Assert(Here = 0)
- If CanGet(1) Then
- Dim ch As Char = Peek(1)
+ Dim ch As Char
+ If Peep(1, ch) Then
Select Case ch
Case "!"c
- If CanGet(2) Then
- Select Case (Peek(2))
+ Dim c2 As Char
+ If Peep(2, c2) Then
+ Select Case c2
Case "-"c
- If NextIs(3,"-"c) Then
+ If NextIs(3, "-"c) Then
Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc)
End If
Case "["c
- If NextAre(3,"CDATA[") Then
+ If NextAre(3, "CDATA[") Then
Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc)
End If
Case "D"c
- If NextAre(3,"OCTYPE") Then
+ If NextAre(3, "OCTYPE") Then
Return XmlMakeBeginDTDToken(precedingTrivia)
End If
End Select
@@ -372,102 +371,100 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return XmlMakeBeginEndElementToken(precedingTrivia, s_scanNoTriviaFunc)
End Select
End If
-
Return XmlMakeLessToken(precedingTrivia)
Case "]"c
- If NextAre(Here+1,"]>") Then
-
+ If NextAre(Here + 1, "]>") Then
' // If valid characters found then return them.
If Here <> 0 Then
- Return XmlMakeTextLiteralToken(Nothing, Here, scratch)
- End If
-
- ' // Create an invalid character data token for the illegal ']]>' sequence
- Return XmlMakeTextLiteralToken(Nothing, 3, ERRID.ERR_XmlEndCDataNotAllowedInContent)
- End If
- GoTo ScanChars
-
- Case "#"c
- ' // Even though # is valid in content, abort xml scanning if the m_State shows and error
- ' // and the line begins with NL WS* # WS* KW
-
- 'TODO: error recovery - how can we do ths?
-
- 'If m_State.m_IsXmlError Then
- ' MakeXmlCharToken(tokens.tkXmlCharData, Here - m_InputStreamPosition, IsAllWhitespace)
- ' m_InputStreamPosition = Here
-
- ' Dim sharp As Token = MakeToken(tokens.tkSharp, 1)
- ' m_InputStreamPosition += 1
-
- ' While (m_InputStream(m_InputStreamPosition) = " "c OrElse m_InputStream(m_InputStreamPosition) = CHARACTER_TABULATION)
- ' m_InputStreamPosition += 1
- ' End While
-
- ' ScanXmlQName()
-
- ' Dim restart As Token = CheckXmlForStatement()
-
- ' If restart IsNot Nothing Then
- ' ' // Abort Xml - Found Keyword space at the beginning of the line
- ' AbandonTokens(restart)
- ' m_State.Init(LexicalState.VB)
- ' MakeToken(tokens.tkXmlAbort, 0)
- ' Return
- ' End If
-
- ' AbandonTokens(sharp)
- ' Here = m_InputStreamPosition
- 'End If
- GoTo ScanChars
+ Return XmlMakeTextLiteralToken(Nothing, Here, scratch)
+ End If
- Case "%"c
-
- 'TODO: error recovery. We cannot do this.
- 'If there is all whitespace after ">", it will be scanned as insignificant,
- 'but in this case it is significant.
- 'Also as far as I can see Dev10 does not resync on "%>" text anyways.
-
- '' // Even though %> is valid in pcdata. When inside of an embedded expression
- '' // return this sequence separately so that the xml literal completion code can
- '' // easily detect the end of an embedded expression that may be temporarily hidden
- '' // by a new element. i.e. <%= %>
-
- 'If CanGetCharAtOffset(Here + 1) AndAlso _
- ' PeekAheadChar(Here + 1) = ">"c Then
-
- ' ' // If valid characters found then return them.
- ' If Here <> 0 Then
- ' Return XmlMakeCharDataToken(Nothing, Here, New String(value.ToArray))
- ' End If
-
- ' ' // Create a special pcdata token for the possible tkEndXmlEmbedded
- ' Return XmlMakeCharDataToken(Nothing, 2, "%>")
- 'Else
- ' IsAllWhitespace = False
- ' value.Add("%"c)
- ' Here += 1
- 'End If
- 'Continue While
- GoTo ScanChars
- Case Else
+ ' // Create an invalid character data token for the illegal ']]>' sequence
+ Return XmlMakeTextLiteralToken(Nothing, 3, ERRID.ERR_XmlEndCDataNotAllowedInContent)
+ End If
+ GoTo ScanChars
+
+ Case "#"c
+ ' // Even though # is valid in content, abort xml scanning if the m_State shows and error
+ ' // and the line begins with NL WS* # WS* KW
+
+ 'TODO: error recovery - how can we do ths?
+
+ 'If m_State.m_IsXmlError Then
+ ' MakeXmlCharToken(tokens.tkXmlCharData, Here - m_InputStreamPosition, IsAllWhitespace)
+ ' m_InputStreamPosition = Here
+
+ ' Dim sharp As Token = MakeToken(tokens.tkSharp, 1)
+ ' m_InputStreamPosition += 1
+
+ ' While (m_InputStream(m_InputStreamPosition) = " "c OrElse m_InputStream(m_InputStreamPosition) = CHARACTER_TABULATION)
+ ' m_InputStreamPosition += 1
+ ' End While
+
+ ' ScanXmlQName()
+
+ ' Dim restart As Token = CheckXmlForStatement()
+
+ ' If restart IsNot Nothing Then
+ ' ' // Abort Xml - Found Keyword space at the beginning of the line
+ ' AbandonTokens(restart)
+ ' m_State.Init(LexicalState.VB)
+ ' MakeToken(tokens.tkXmlAbort, 0)
+ ' Return
+ ' End If
+
+ ' AbandonTokens(sharp)
+ ' Here = m_InputStreamPosition
+ 'End If
+ GoTo ScanChars
+
+ Case "%"c
+
+ 'TODO: error recovery. We cannot do this.
+ 'If there is all whitespace after ">", it will be scanned as insignificant,
+ 'but in this case it is significant.
+ 'Also as far as I can see Dev10 does not resync on "%>" text anyways.
+
+ '' // Even though %> is valid in pcdata. When inside of an embedded expression
+ '' // return this sequence separately so that the xml literal completion code can
+ '' // easily detect the end of an embedded expression that may be temporarily hidden
+ '' // by a new element. i.e. <%= %>
+
+ 'If CanGetCharAtOffset(Here + 1) AndAlso _
+ ' PeekAheadChar(Here + 1) = ">"c Then
+
+ ' ' // If valid characters found then return them.
+ ' If Here <> 0 Then
+ ' Return XmlMakeCharDataToken(Nothing, Here, New String(value.ToArray))
+ ' End If
+
+ ' ' // Create a special pcdata token for the possible tkEndXmlEmbedded
+ ' Return XmlMakeCharDataToken(Nothing, 2, "%>")
+ 'Else
+ ' IsAllWhitespace = False
+ ' value.Add("%"c)
+ ' Here += 1
+ 'End If
+ 'Continue While
+ GoTo ScanChars
+ Case Else
ScanChars:
- ' // Check characters are valid
- IsAllWhitespace = False
- Dim xmlCh = ScanXmlChar(Here)
-
- If xmlCh.Length = 0 Then
- ' bad char
- If Here > 0 Then
- Return XmlMakeTextLiteralToken(Nothing, Here, scratch)
- Else
- Return XmlMakeBadToken(Nothing, 1, ERRID.ERR_IllegalChar)
+ ' // Check characters are valid
+ IsAllWhitespace = False
+ Dim xmlCh = ScanXmlChar(Here)
+
+ If xmlCh.Length = 0 Then
+ ' bad char
+ If Here > 0 Then
+ Return XmlMakeTextLiteralToken(Nothing, Here, scratch)
+ Else
+ Return XmlMakeBadToken(Nothing, 1, ERRID.ERR_IllegalChar)
+ End If
End If
- End If
- xmlCh.AppendTo(scratch)
- Here += xmlCh.Length
+ xmlCh.AppendTo(scratch)
+ Here += xmlCh.Length
End Select
End While
@@ -492,69 +489,66 @@ ScanChars:
End If
Dim Here = 0
- While CanGet(Here)
- Dim c As Char = Peek(Here)
- Select Case (c)
+ Dim c As Char
+ While Peep(Here, c)
+ Select Case c
Case CARRIAGE_RETURN, LINE_FEED
Return XmlMakeCommentToken(precedingTrivia, Here + LengthOfLineBreak(c, Here))
Case "-"c
- If NextIs(Here + 1,"-"c) Then
-
+ If NextIs(Here + 1, "-"c) Then
' // --> terminates an Xml comment but otherwise -- is an illegal character sequence.
' // The scanner will always returns "--" as a separate comment data string and the
' // the semantics will error if '--' is ever found.
' // Return valid characters up to the --
If Here > 0 Then
- Return XmlMakeCommentToken(precedingTrivia, Here)
- End If
-
- If CanGet(Here + 2) Then
+ Return XmlMakeCommentToken(precedingTrivia, Here)
+ End If
- c = Peek(Here + 2)
- Here += 2
- ' // if > is not found then this is an error. Return the -- string
+ If Peep(Here + 2, c) Then
+ Here += 2
+ ' // if > is not found then this is an error. Return the -- string
- If c <> ">"c Then
- Return XmlMakeCommentToken(precedingTrivia, 2)
+ If c <> ">"c Then
+ Return XmlMakeCommentToken(precedingTrivia, 2)
- ' TODO: we cannot do the following
+ ' TODO: we cannot do the following
- ' // For better error recovery, allow -> to terminate the comment.
- ' // This works because the -> terminates only when the invalid --
- ' // is returned.
+ ' // For better error recovery, allow -> to terminate the comment.
+ ' // This works because the -> terminates only when the invalid --
+ ' // is returned.
- 'If Here + 1 < m_InputStreamEnd AndAlso _
- ' m_InputStream(Here) = "-"c AndAlso _
- ' m_InputStream(Here + 1) = ">"c Then
+ 'If Here + 1 < m_InputStreamEnd AndAlso _
+ ' m_InputStream(Here) = "-"c AndAlso _
+ ' m_InputStream(Here + 1) = ">"c Then
- ' Here += 1
- 'Else
- ' Continue While
- 'End If
- Else
- Return XmlMakeEndCommentToken(precedingTrivia)
+ ' Here += 1
+ 'Else
+ ' Continue While
+ 'End If
+ Else
+ Return XmlMakeEndCommentToken(precedingTrivia)
+ End If
End If
End If
- End If
- GoTo ScanChars
+ GoTo ScanChars
- Case Else
+ Case Else
ScanChars:
- Dim xmlCh = ScanXmlChar(Here)
- If xmlCh.Length <> 0 Then
- Here += xmlCh.Length
- Continue While
- End If
+ Dim xmlCh = ScanXmlChar(Here)
+ If xmlCh.Length <> 0 Then
+ Here += xmlCh.Length
+ Continue While
+ End If
- ' bad char
- If Here > 0 Then
- Return XmlMakeCommentToken(precedingTrivia, Here)
- Else
- Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalChar)
- End If
+ ' bad char
+ If Here > 0 Then
+ Return XmlMakeCommentToken(precedingTrivia, Here)
+ Else
+ Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalChar)
+ End If
End Select
End While
@@ -583,18 +577,16 @@ ScanChars:
Dim scratch = GetScratch()
Dim Here = 0
-
- While CanGet(Here)
- Dim c As Char = Peek(Here)
- Select Case (c)
-
+ Dim c As Char
+ While Peep(Here, c)
+ Select Case c
Case CARRIAGE_RETURN, LINE_FEED
Here = SkipLineBreak(c, Here)
scratch.Append(LINE_FEED)
Return XmlMakeCDataToken(precedingTrivia, Here, scratch)
Case "]"c
- If NextAre(Here+1,"]>") Then
+ If NextAre(Here + 1, "]>") Then
'// If valid characters found then return them.
If Here <> 0 Then
@@ -647,11 +639,10 @@ ScanChars:
Dim precedingTrivia = _triviaListPool.Allocate(Of VisualBasicSyntaxNode)()
Dim result As SyntaxToken
-
- If state = ScannerState.StartProcessingInstruction AndAlso CanGet() Then
+ Dim c As Char
+ If state = ScannerState.StartProcessingInstruction AndAlso Peep(c) Then
' // Whitespace
' // S ::= (#x20 | #x9 | #xD | #xA)+
- Dim c = Peek()
Select Case c
Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION
Dim wsTrivia = ScanXmlTrivia(c)
@@ -660,8 +651,7 @@ ScanChars:
End If
Dim Here = 0
- While CanGet(Here)
- Dim c As Char = Peek(Here)
+ While Peep(Here, c)
Select Case (c)
Case CARRIAGE_RETURN, LINE_FEED
@@ -670,7 +660,6 @@ ScanChars:
Case "?"c
If NextIs(Here + 1, ">"c) Then
-
'// If valid characters found then return them.
If Here <> 0 Then
result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here)
@@ -720,9 +709,8 @@ CleanUp:
' // Misc ::= Comment | PI | S
Dim precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode) = Nothing
- While CanGet()
- Dim c As Char = Peek()
-
+ Dim c As Char
+ While Peep(c)
Select Case (c)
' // Whitespace
' // S ::= (#x20 | #x9 | #xD | #xA)+
@@ -732,17 +720,17 @@ CleanUp:
precedingTrivia = ScanXmlTrivia(c)
Case "<"c
- If CanGet(1) Then
- Dim ch As Char = Peek(1)
+ Dim ch As Char
+ If Peep(1, ch) Then
Select Case ch
Case "!"c
- If NextAre(2,"--") Then
+ If NextAre(2, "--") Then
Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc)
- ElseIf NextAre(2,"DOCTYPE") Then
+ ElseIf NextAre(2, "DOCTYPE") Then
Return XmlMakeBeginDTDToken(precedingTrivia)
End If
Case "%"c
- If NextIs(2,"="c) Then
+ If NextIs(2, "="c) Then
Return XmlMakeBeginEmbeddedToken(precedingTrivia)
End If
Case "?"c
@@ -752,7 +740,7 @@ CleanUp:
Return XmlMakeLessToken(precedingTrivia)
- ' TODO: review
+ ' TODO: review
' If Not m_State.m_ScannedElement OrElse c = "?"c OrElse c = "!"c Then
' ' // Remove tEOL from token ring if any exists
@@ -793,14 +781,12 @@ CleanUp:
' This can never happen as this token cannot cross lines.
Debug.Assert(Not (IsScanningXmlDoc AndAlso IsAtNewLine()))
+ Dim c As Char
Dim Here = 0
Dim scratch = GetScratch()
- While CanGet(Here)
- Dim c As Char = Peek(Here)
-
+ While Peep(Here, c)
Select Case (c)
-
Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION
If Here > 0 Then
Return XmlMakeAttributeDataToken(Nothing, Here, scratch)
@@ -888,9 +874,8 @@ ScanChars:
Dim Here = 0
Dim scratch = GetScratch()
-
- While CanGet(Here)
- Dim c As Char = Peek(Here)
+ Dim c As Char
+ While Peep(Here, c)
If c = terminatingChar Or c = altTerminatingChar Then
If Here > 0 Then
result = XmlMakeAttributeDataToken(precedingTrivia, Here, scratch)
@@ -923,7 +908,7 @@ ScanChars:
GoTo CleanUp
Else
' report unexpected <%= in a special way.
- If NextAre(1,"%=") Then
+ If NextAre(1, "%=") Then
Dim errEmbedStart = XmlMakeAttributeDataToken(precedingTrivia, 3, "<%=")
Dim errEmberinfo = ErrorFactory.ErrorInfo(ERRID.ERR_QuotedEmbeddedExpression)
result = DirectCast(errEmbedStart.SetDiagnostics({errEmberinfo}), SyntaxToken)
@@ -984,17 +969,15 @@ CleanUp:
End Function
'''
- ''' 0 - not a surrogate, 2 - is valid surrogate
+ ''' 0 - not a surrogate, 2 - is valid surrogate
''' 1 is an error
'''
Private Function ScanSurrogatePair(c1 As Char, Here As Integer) As XmlCharResult
Debug.Assert(Here >= 0)
Debug.Assert(CanGet(Here))
Debug.Assert(Peek(Here) = c1)
-
- If IsHighSurrogate(c1) AndAlso CanGet(Here + 1) Then
- Dim c2 = Peek(Here + 1)
-
+ Dim c2 As Char
+ If IsHighSurrogate(c1) AndAlso Peep(Here + 1, c2) Then
If IsLowSurrogate(c2) Then
Return New XmlCharResult(c1, c2)
End If
@@ -1003,7 +986,7 @@ CleanUp:
Return Nothing
End Function
- ' contains result of Xml char scanning.
+ ' contains result of Xml char scanning.
Friend Structure XmlCharResult
Friend ReadOnly Length As Integer
Friend ReadOnly Char1 As Char
@@ -1033,9 +1016,9 @@ CleanUp:
Private Function ScanXmlChar(Here As Integer) As XmlCharResult
Debug.Assert(Here >= 0)
- Debug.Assert(CanGet(Here))
-
- Dim c = Peek(Here)
+ Dim c As Char
+ Dim ok = Peep(Here, c)
+ Debug.Assert(ok)
If Not isValidUtf16(c) Then
Return Nothing
@@ -1066,9 +1049,8 @@ CleanUp:
'TODO - Fix ScanXmlNCName to conform to XML spec instead of old loose scanning.
- While CanGet(Here)
- Dim c As Char = Peek(Here)
-
+ Dim c As Char
+ While Peep(Here, c)
Select Case (c)
Case ":"c, " "c, CHARACTER_TABULATION, LINE_FEED, CARRIAGE_RETURN,
@@ -1130,13 +1112,10 @@ CreateNCNameToken:
End Function
Private Function ScanXmlReference(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As XmlTextTokenSyntax
- Debug.Assert(CanGet)
- Debug.Assert(Peek() = "&"c)
-
+ Debug.Assert(NextIs(0, "&"c))
+ Dim c, c1 As Char
' skip 1 char for "&"
- If CanGet(1) Then
- Dim c As Char = Peek(1)
-
+ If Peep(1, c) Then
Select Case (c)
Case "#"c
Dim Here = 2 ' skip ""
@@ -1150,7 +1129,7 @@ CreateNCNameToken:
value = Intern({result.Char1, result.Char2})
End If
- If CanGet(Here) AndAlso Peek(Here) = ";"c Then
+ If NextIs(Here, ";"c) Then
Return XmlMakeEntityLiteralToken(precedingTrivia, Here + 1, value)
Else
Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, Here, value)
@@ -1162,9 +1141,8 @@ CreateNCNameToken:
Case "a"c
' // &
' // '
-
- If CanGet(4) AndAlso NextAre(2,"mp") Then
- If Peek(4) = ";"c Then
+ If Peep(4, c1) AndAlso NextAre(2, "mp") Then
+ If c1 = ";"c Then
Return XmlMakeAmpLiteralToken(precedingTrivia)
Else
Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 4, "&")
@@ -1172,9 +1150,8 @@ CreateNCNameToken:
Return DirectCast(noSemicolon.SetDiagnostics({noSemicolonError}), XmlTextTokenSyntax)
End If
- ElseIf CanGet(5) AndAlso NextAre(2,"pos") Then
-
- If Peek(5) = ";"c Then
+ ElseIf Peep(5, c1) AndAlso NextAre(2, "pos") Then
+ If c1 = ";"c Then
Return XmlMakeAposLiteralToken(precedingTrivia)
Else
Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 5, "'")
@@ -1185,10 +1162,8 @@ CreateNCNameToken:
Case "l"c
' // <
-
- If CanGet(3) AndAlso NextIs(2,"t"c) Then
-
- If Peek(3) = ";"c Then
+ If Peep(3, c1) AndAlso NextIs(2, "t"c) Then
+ If c1 = ";"c Then
Return XmlMakeLtLiteralToken(precedingTrivia)
Else
Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 3, "<")
@@ -1199,10 +1174,8 @@ CreateNCNameToken:
Case "g"c
' // >
-
- If CanGet(3) AndAlso NextIs(2,"t"c) Then
-
- If Peek(3) = ";"c Then
+ If Peep(3, c1) AndAlso NextIs(2, "t"c) Then
+ If c1 = ";"c Then
Return XmlMakeGtLiteralToken(precedingTrivia)
Else
Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 3, ">")
@@ -1213,10 +1186,8 @@ CreateNCNameToken:
Case "q"c
' // "
-
- If CanGet(5) AndAlso NextAre(2,"uot") Then
-
- If Peek(5) = ";"c Then
+ If Peep(5, c1) AndAlso NextAre(2, "uot") Then
+ If c1 = ";"c Then
Return XmlMakeQuotLiteralToken(precedingTrivia)
Else
Dim noSemicolon = XmlMakeEntityLiteralToken(precedingTrivia, 5, """")
@@ -1236,21 +1207,18 @@ CreateNCNameToken:
Private Function ScanXmlCharRef(ByRef index As Integer) As XmlCharResult
Debug.Assert(index >= 0)
-
- If Not CanGet(index) Then
+ Dim ch As Char
+ If Not Peep(index, ch) Then
Return Nothing
End If
' cannot reuse Scratch as this can be used in a nested call.
Dim charRefSb As New StringBuilder
Dim Here = index
-
- Dim ch = Peek(Here)
If ch = "x"c Then
Here += 1
- While CanGet(Here)
- ch = Peek(Here)
+ While Peep(Here, ch)
If XmlCharType.IsHexDigit(ch) Then
charRefSb.Append(ch)
Else
@@ -1266,13 +1234,9 @@ CreateNCNameToken:
Return result
End If
Else
- While CanGet(Here)
- ch = Peek(Here)
- If XmlCharType.IsDigit(ch) Then
- charRefSb.Append(ch)
- Else
- Exit While
- End If
+
+ While Peep(Here, ch) AndAlso XmlCharType.IsDigit(ch)
+ charRefSb.Append(ch)
Here += 1
End While
If charRefSb.Length > 0 Then
@@ -1285,5 +1249,7 @@ CreateNCNameToken:
End If
Return Nothing
End Function
+
End Class
+
End Namespace
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb
index 2f402543c7229..6231790bcd4c2 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/TokenFactories.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Compare Binary
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb b/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb
index 7080f6ad81ff8..d6af5c85865ec 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/TokenStream.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Compare Binary
Option Strict On
@@ -151,8 +151,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'''
''' Gets current reusable syntax node.
- ''' If node is returned its start will be aligned with the start of current token.
- ''' NOTE: Line offset may not match start of current token because of lookahead.
+ ''' If node is returned its start will be aligned with the start of current token.
+ ''' NOTE: Line offset may not match start of current token because of lookahead.
'''
Friend Overridable Function GetCurrentSyntaxNode() As VisualBasicSyntaxNode
Return Nothing
@@ -161,12 +161,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'''
''' Indicates that previously returned node has been consumed
''' and scanner needs to advance by the size of the node.
- '''
+ '''
''' NOTE: the advancement is done relative to the start of the current token.
- ''' Line offset may not match start of current token because of lookahead.
- '''
- ''' This operation will discard lookahead tokens and reset preprocessor state
- ''' to the state of current token.
+ ''' Line offset may not match start of current token because of lookahead.
+ '''
+ ''' This operation will discard lookahead tokens and reset preprocessor state
+ ''' to the state of current token.
'''
Friend Overridable Sub MoveToNextSyntaxNode()
' do not use prev token after consuming a nonterminal
@@ -180,14 +180,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
'''
''' Indicates that previously returned node has been consumed
''' and scanner needs to advance by the size of the node.
- '''
+ '''
''' NOTE: the advancement is done relative to the _lineBufferOffset.
''' Line offset will likely not match start of current token because this operation
''' is done while constructing the content of current token.
- '''
+ '''
''' NOTE: This operation assumes that there is no tokens read ahead.
- '''
- ''' NOTE: This operation does not change preprocessor state.
+ '''
+ ''' NOTE: This operation does not change preprocessor state.
''' The assumption is that it is responsibility of the node consumer to update preprocessor
''' state if needed when using nodes that change preprocessor state.
'''
@@ -236,7 +236,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
If state <> _currentToken.State Then
' this is a special case for switching from VB to Xml
- ' we need to keep preceding trivia as it was scanned
+ ' we need to keep preceding trivia as it was scanned
If _currentToken.State = ScannerState.VB AndAlso state = ScannerState.Content Then
Dim vbTk = GetCurrentToken()
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb
index 175b5145f172f..54af92bbe03e4 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlDocComments.vb
@@ -49,9 +49,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function TryScanXmlDocComment(tList As SyntaxListBuilder) As Boolean
Debug.Assert(IsAtNewLine)
-
' leading whitespace until we see ''' should be regular whitespace
- If CanGet() AndAlso IsWhitespace(Peek()) Then
+ Dim c As Char
+ If Peep(c) AndAlso IsWhitespace(c) Then
Dim ws = ScanWhitespace()
tList.Add(ws)
End If
@@ -62,8 +62,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
#End If
Dim restorePoint = CreateRestorePoint()
- ' since we do not have lookahead tokens, this just
- ' resets current token to _lineBufferOffset
+ ' since we do not have lookahead tokens, this just
+ ' resets current token to _lineBufferOffset
Me.GetNextTokenInState(ScannerState.Content)
Dim currentNonterminal = Me.GetCurrentSyntaxNode()
@@ -79,23 +79,23 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Me.IsScanningXmlDoc = True
Me._isStartingFirstXmlDocLine = True
- ' NOTE: Documentation comment syntax trivia must have at least one child xml node, because
+ ' NOTE: Documentation comment syntax trivia must have at least one child xml node, because
' all the ['''] trivia are created as leading trivia for appropriate tokens.
' This means that we have to create at least one XmlText having trailing
' EOL to represent an empty documentation comment: [''']
'
' The problem with this approach is that in presence of some errors (like
' not closed XML tags) we create missing tokens needed to represent the nodes
- ' *after* that last of the doc comment trivia, that means all the locations
+ ' *after* that last of the doc comment trivia, that means all the locations
' of created diagnostics will land on the first character of the next line
' after documentation comment
'
- ' To workaround this we parse XML nodes in two phases:
- ' - in the first phase we detect the last DocCommentLineBreak and create
- ' end-of-xml token instead; this should force all diagnostics to be
+ ' To workaround this we parse XML nodes in two phases:
+ ' - in the first phase we detect the last DocCommentLineBreak and create
+ ' end-of-xml token instead; this should force all diagnostics to be
' reported on the next token location;
- ' - in the second phase we continue parsing XML nodes but don't create
- ' end-of-xml token which should just result in parsing one single node
+ ' - in the second phase we continue parsing XML nodes but don't create
+ ' end-of-xml token which should just result in parsing one single node
' of XmlText type containing EOL;
' Then we merge the results and create resulting DocumentationCommentTrivia
@@ -106,7 +106,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' The second phase
Me._endOfXmlInsteadOfLastDocCommentLineBreak = False
If nodes.Count = 0 AndAlso parser.CurrentToken.Kind = SyntaxKind.EndOfXmlToken Then
- ' This must be an empty documentation comment, we need to reset scanner so
+ ' This must be an empty documentation comment, we need to reset scanner so
' that the doc comment exterior trivia ([''']) lands on the final XmlNode
ResetLineBufferOffset()
@@ -147,14 +147,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
' lexes (ws)'''
Private Function TrySkipXmlDocMarker(ByRef len As Integer) As Boolean
+ Dim c As Char
Dim Here = len
- While CanGet(Here)
- Dim c = Peek(Here)
- If IsWhitespace(c) Then
- Here += 1
- Else
- Exit While
- End If
+ While Peep(Here, c) AndAlso IsWhitespace(c)
+ Here += 1
End While
If StartsXmlDoc(Here) Then
@@ -186,7 +182,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim len = 0
Do
- If c = " "c OrElse c = CHARACTER_TABULATION Then
+ If c.IsFrom(" "c, CHARACTER_TABULATION) Then
len += 1
ElseIf IsNewLine(c) Then
@@ -240,9 +236,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Dim Here As Integer = 0
Dim scratch = GetScratch()
-
- While CanGet(Here)
- Dim c As Char = Peek(Here)
+ Dim c As Char
+ While Peep(Here, c)
Select Case (c)
Case CARRIAGE_RETURN, LINE_FEED
@@ -255,7 +250,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
If _endOfXmlInsteadOfLastDocCommentLineBreak Then
Dim tempHere As Integer = Here
If Not TrySkipXmlDocMarker(tempHere) Then
- ' NOTE: we need to reset the buffer so that precedingTrivia
+ ' NOTE: we need to reset the buffer so that precedingTrivia
' lands on the next token
ResetLineBufferOffset()
Return SyntaxFactory.Token(Nothing, SyntaxKind.EndOfXmlToken, Nothing, String.Empty)
@@ -282,22 +277,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End If
Debug.Assert(Here = 0)
- If CanGet(1) Then
- Dim ch As Char = Peek(1)
- Select Case ch
+ If Peep(1, c) Then
+ Select Case c
Case "!"c
- If CanGet(2) Then
- Select Case (Peek(2))
+ If Peep(2, c) Then
+ Select Case c
Case "-"c
- If NextIs(3,"-"c) Then
+ If NextIs(3, "-"c) Then
Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc)
End If
Case "["c
- If NextAre(3,"CDATA[") Then
+ If NextAre(3, "CDATA[") Then
Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc)
End If
Case "D"c
- If NextAre(3,"OCTYPE") Then
+ If NextAre(3, "OCTYPE") Then
Return XmlMakeBeginDTDToken(precedingTrivia)
End If
End Select
@@ -325,7 +319,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
GoTo ScanChars
Case Else
ScanChars:
- ' // Check characters are valid
+ ' // Check characters are valid
Dim xmlCh = ScanXmlChar(Here)
If xmlCh.Length = 0 Then
@@ -370,10 +364,10 @@ ScanChars:
precedingTrivia.Add(xDocTrivia)
End If
- If state = ScannerState.StartProcessingInstruction AndAlso CanGet() Then
+ Dim c As Char
+ If state = ScannerState.StartProcessingInstruction AndAlso Peep(c) Then
' // Whitespace
' // S ::= (#x20 | #x9 | #xD | #xA)+
- Dim c = Peek()
Select Case c
Case CARRIAGE_RETURN, LINE_FEED, " "c, CHARACTER_TABULATION
Dim offsets = CreateOffsetRestorePoint()
@@ -387,8 +381,7 @@ ScanChars:
End If
Dim Here = 0
- While CanGet(Here)
- Dim c As Char = Peek(Here)
+ While Peep(Here, c)
Select Case (c)
Case CARRIAGE_RETURN, LINE_FEED
@@ -396,8 +389,7 @@ ScanChars:
GoTo CleanUp
Case "?"c
- If NextIs(Here + 1,">"c) Then
-
+ If NextIs(Here + 1, ">"c) Then
'// If valid characters found then return them.
If Here <> 0 Then
result = XmlMakeProcessingInstructionToken(precedingTrivia.ToList, Here)
@@ -461,7 +453,8 @@ CleanUp:
precedingTrivia = New SyntaxList(Of VisualBasicSyntaxNode)(xDocTrivia)
End If
- While CanGet()
+ Dim c As Char
+ While Peep(c)
If Not precedingTrivia.Any AndAlso IsAtNewLine() AndAlso Not Me._doNotRequireXmlDocCommentPrefix Then
' this would indicate that we looked at Trivia, but did not find
' XmlDoc prefix (or we would not be at the line start)
@@ -469,8 +462,6 @@ CleanUp:
Return MakeEofToken(precedingTrivia)
End If
- Dim c As Char = Peek()
-
Select Case (c)
' // Whitespace
' // S ::= (#x20 | #x9 | #xD | #xA)+
@@ -506,22 +497,21 @@ CleanUp:
Return XmlMakeDoubleQuoteToken(precedingTrivia, c, isOpening:=True)
Case "<"c
- If CanGet(1) Then
- Dim ch As Char = Peek(1)
- Select Case ch
+ If Peep(1, c) Then
+ Select Case c
Case "!"c
- If CanGet(2) Then
- Select Case (Peek(2))
+ If Peep(2, c) Then
+ Select Case c
Case "-"c
- If NextIs(3,"-"c) Then
+ If NextIs(3, "-"c) Then
Return XmlMakeBeginCommentToken(precedingTrivia, s_scanNoTriviaFunc)
End If
Case "["c
- If NextAre(3,"CDATA[") Then
+ If NextAre(3, "CDATA[") Then
Return XmlMakeBeginCDataToken(precedingTrivia, s_scanNoTriviaFunc)
End If
Case "D"c
- If NextAre(3,"OCTYPE") Then
+ If NextAre(3, "OCTYPE") Then
Return XmlMakeBeginDTDToken(precedingTrivia)
End If
End Select
@@ -550,11 +540,7 @@ CleanUp:
Case ")"c
Return XmlMakeRightParenToken(precedingTrivia)
- Case "!"c,
- ";"c,
- "#"c,
- ","c,
- "}"c
+ Case "!"c, ";"c, "#"c, ","c, "}"c
Return XmlMakeBadToken(precedingTrivia, 1, ERRID.ERR_IllegalXmlNameChar)
Case ":"c
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb
index 5ed11a1fcb293..fc5767cffcf6d 100644
--- a/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb
+++ b/src/Compilers/VisualBasic/Portable/Scanner/XmlTokenFactories.vb
@@ -1,7 +1,7 @@
' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
'-----------------------------------------------------------------------------
-' Contains the definition of the Scanner, which produces tokens from text
+' Contains the definition of the Scanner, which produces tokens from text
'-----------------------------------------------------------------------------
Option Compare Binary
@@ -298,7 +298,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function XmlMakeBeginEndElementToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax
Debug.Assert(NextAre(""))
-
AdvanceChar(2)
Dim followingTrivia = scanTrailingTrivia()
Return MakePunctuationToken(SyntaxKind.LessThanSlashToken, "", precedingTrivia, followingTrivia)
@@ -306,7 +305,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function XmlMakeEndEmptyElementToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As PunctuationSyntax
Debug.Assert(NextAre("/>"))
-
AdvanceChar(2)
Return MakePunctuationToken(SyntaxKind.SlashGreaterThanToken, "/>", precedingTrivia, Nothing)
End Function
@@ -319,11 +317,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
End Function
Private Function XmlMakeEndEmbeddedToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax
- Debug.Assert(Peek() = "%"c OrElse Peek() = FULLWIDTH_PERCENT_SIGN)
+ Dim c = Peek
+ Debug.Assert(c.IsFrom("%"c, FULLWIDTH_PERCENT_SIGN))
Debug.Assert(Peek(1) = ">"c)
Dim spelling As String
- If Peek() = "%"c Then
+ If c = "%"c Then
AdvanceChar(2)
spelling = "%>"
Else
@@ -354,7 +353,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Private Function XmlMakeCloseBracketToken(state As ScannerState, precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode)) As BadTokenSyntax
Debug.Assert(Peek() = "]"c)
-
Return XmlMakeBadToken(SyntaxSubKind.CloseBracketToken, precedingTrivia, 1, If(state = ScannerState.DocType, ERRID.ERR_DTDNotSupported, ERRID.ERR_IllegalXmlNameChar))
End Function
#End Region
@@ -413,7 +411,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
#Region "CData"
Private Function XmlMakeBeginCDataToken(precedingTrivia As SyntaxList(Of VisualBasicSyntaxNode), scanTrailingTrivia As ScanTriviaFunc) As PunctuationSyntax
Debug.Assert(NextAre(" SyntaxKind.None Then
- Return String.Format("[{0}]", identifier)
+ Return $"[{identifier}]"
End If
' The minimal flag is e.g. used by the service layer when generating code (example: simplify name).
@@ -113,13 +113,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
If Me.IsMinimizing Then
Dim contextualKeywordKind As SyntaxKind = SyntaxFacts.GetContextualKeywordKind(identifier)
- ' Leading implicit line continuation is allowed before query operators (Aggregate, Distinct, From, Group By,
- ' Group Join, Join, Let, Order By, Select, Skip, Skip While, Take, Take While, Where, In, Into, On, Ascending,
+ ' Leading implicit line continuation is allowed before query operators (Aggregate, Distinct, From, Group By,
+ ' Group Join, Join, Let, Order By, Select, Skip, Skip While, Take, Take While, Where, In, Into, On, Ascending,
' and Descending).
' If the current identifier is one of these keywords, we need to escape them to avoid parsing issues if the
' previous line was a query expression.
'
- ' In addition to the query operators, we need to escape the identifier "preserve" to avoid ambiguities
+ ' In addition to the query operators, we need to escape the identifier "preserve" to avoid ambiguities
' inside of a dim statement (dim [preserve].ArrayName(1))
Select Case contextualKeywordKind
Case SyntaxKind.AggregateKeyword,
@@ -139,7 +139,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
SyntaxKind.AscendingKeyword,
SyntaxKind.DescendingKeyword,
SyntaxKind.PreserveKeyword
- Return String.Format("[{0}]", identifier)
+ Return $"[{identifier}]"
End Select
End If
diff --git a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb
index 0d6f65a80c7f4..2bcd96e563d3f 100644
--- a/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb
+++ b/src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplayVisitor_Minimal.vb
@@ -16,7 +16,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
Debug.Assert(symbol.ContainingNamespace IsNot Nothing OrElse symbol.IsGlobalNamespace)
' NOTE(cyrusn): We only call this once we've already checked if there is an alias that
- ' corresponds to this namespace.
+ ' corresponds to this namespace.
If symbol.IsGlobalNamespace Then
' nothing to add for global namespace itself