Merge pull request #207 from BenPH/fix-ddot-lexing

Fix tokenization of numbers followed by `..`

Author: pfitzseb

src/lexer.jl

@@ -629,12 +629,13 @@ function lex_digit(l::Lexer, kind)
     accept_number(l, isdigit)
     pc,ppc = dpeekchar(l)
     if pc == '.'
-        if kind === Tokens.FLOAT
+        if ppc == '.'
+            # Number followed by .. or ...
+            return emit(l, kind)
+        elseif kind === Tokens.FLOAT
             # If we enter the function with kind == FLOAT then a '.' has been parsed.
             readchar(l)
             return emit_error(l, Tokens.INVALID_NUMERIC_CONSTANT)
-        elseif ppc == '.'
-            return emit(l, kind)
         elseif is_operator_start_char(ppc) && ppc !== ':'
             readchar(l)
             return emit_error(l)
@@ -703,7 +704,9 @@ function lex_digit(l::Lexer, kind)
             readchar(l)
             !(ishex(ppc) || ppc == '.') && return emit_error(l, Tokens.INVALID_NUMERIC_CONSTANT)
             accept_number(l, ishex)
-            if accept(l, '.')
+            pc,ppc = dpeekchar(l)
+            if pc == '.' && ppc != '.'
+                readchar(l)
                 accept_number(l, ishex)
                 isfloat = true
             end

test/lexer.jl

@@ -358,7 +358,7 @@ end
     @test tok("1234x", 2).kind        == T.IDENTIFIER
 end
 
-@testset "floats with trailing `.` " begin
+@testset "numbers with trailing `.` " begin
     @test tok("1.0").kind == Tokens.FLOAT
     @test tok("1.a").kind == Tokens.FLOAT
     @test tok("1.(").kind == Tokens.FLOAT
@@ -373,7 +373,10 @@ end
     @test tok("1.").kind == Tokens.FLOAT
     @test tok("1.\"text\" ").kind == Tokens.FLOAT
 
-    @test tok("1..").kind  == Tokens.INTEGER
+    @test toks("1..")    == ["1"=>Tokens.INTEGER,   ".."=>Tokens.DDOT]
+    @test toks(".1..")   == [".1"=>Tokens.FLOAT,    ".."=>Tokens.DDOT]
+    @test toks("0x01..") == ["0x01"=>Tokens.HEX_INT, ".."=>Tokens.DDOT]
+
     @test T.kind.(collect(tokenize("1f0./1"))) == [T.FLOAT, T.OP, T.INTEGER, T.ENDMARKER]
 end