Merge pull request #108 from davidanthoff/escaped-strings

Fix escaped strings handling

Author: davidanthoff

src/csv.jl

@@ -44,12 +44,13 @@ end
 
 optionsiter(opts::AbstractVector, header) = optionsiter(opts)
 
-tofield(f::AbstractField, opts) = f
-tofield(f::AbstractToken, opts) = Field(f)
-tofield(f::StringToken, opts) = Field(Quoted(f, opts.quotechar, opts.escapechar))
-tofield(f::Type, opts) = tofield(fromtype(f), opts)
-tofield(f::Type{String}, opts) = tofield(fromtype(StrRange), opts)
-tofield(f::DateFormat, opts) = tofield(DateTimeToken(DateTime, f), opts)
+tofield(f::AbstractField, opts, stringarraytype) = f
+tofield(f::AbstractToken, opts, stringarraytype) = Field(f)
+tofield(f::StringToken, opts, stringarraytype) = Field(Quoted(f, opts.quotechar, opts.escapechar))
+tofield(f::Type, opts, stringarraytype) = tofield(fromtype(f), opts, stringarraytype)
+tofield(f::Type{String}, opts, stringarraytype::Type{StringArray}) = tofield(fromtype(StrRange), opts, stringarraytype)
+tofield(f::Type{String}, opts, stringarraytype::Type{Array}) = tofield(fromtype(String), opts, stringarraytype)
+tofield(f::DateFormat, opts, stringarraytype) = tofield(DateTimeToken(DateTime, f), opts, stringarraytype)
 
 """
     csvread(file::Union{String,IO}, delim=','; <arguments>...)
@@ -241,9 +242,8 @@ function _csvread_internal(str::AbstractString, delim=',';
 
     # seed guesses using those from previous file
     guess, pos1 = guesscolparsers(str, canonnames, opts,
-                                  pos, type_detect_rows, 
-                                  colparsers, commentchar,
-                                  nastrings, prev_parsers)
+                                  pos, type_detect_rows, colparsers, stringarraytype, 
+                                  commentchar, nastrings, prev_parsers)
     if isempty(canonnames)
         canonnames = Any[1:length(guess);]
     end
@@ -255,7 +255,7 @@ function _csvread_internal(str::AbstractString, delim=',';
         if !(fieldtype(v) <: StringLike) && prev_parsers !== nothing && !haskey(colspool, c)
             v = isa(v, NAToken) ? v : NAToken(v)
         end
-        p = tofield(v, opts)
+        p = tofield(v, opts, stringarraytype)
         guess[i] = p
     end
 
@@ -276,7 +276,7 @@ function _csvread_internal(str::AbstractString, delim=',';
     current_record[] = rec
 
     if nrows == 0
-        # just an estimate, with some margin        
+        # just an estimate, with some margin
         nrows = ceil(Int, (len-pos) / ((pos1-pos)/max(1, type_detect_rows)) * sqrt(2))
     end
 
@@ -417,7 +417,7 @@ function _csvread_internal(str::AbstractString, delim=',';
 end
 
 function promote_field(failed_str, field, col, err, nastrings, stringtype, stringarraytype, opts)
-    newtoken = guesstoken(failed_str, opts, field.inner, nastrings)
+    newtoken = guesstoken(failed_str, opts, field.inner, nastrings, stringarraytype)
     if newtoken == field.inner
         # no need to change
         return field, col
@@ -478,8 +478,7 @@ function readcolnames(str, opts, pos, colnames)
 end
 
 function guesscolparsers(str::AbstractString, header, opts::LocalOpts, pos::Int,
-                       nrows::Int, colparsers, commentchar=nothing, nastrings=NA_STRINGS,
-                       prevs=nothing)
+                       nrows::Int, colparsers, stringarraytype, commentchar=nothing, nastrings=NA_STRINGS, prevs=nothing)
     # Field type guesses
     guess = []
     prevfields = String[]
@@ -517,7 +516,7 @@ function guesscolparsers(str::AbstractString, header, opts::LocalOpts, pos::Int,
                 error("previous rows had $(length(guess)) fields but row $i2 has $(length(fields))")
             end
             try
-                guess[j] = guesstoken(fields[j], opts, guess[j], nastrings)
+                guess[j] = guesstoken(fields[j], opts, guess[j], nastrings, stringarraytype)
             catch err
                 println(stderr, "Error while guessing a common type for column $j")
                 println(stderr, "new value: $(fields[j]), prev guess was: $(guess[j])")
@@ -534,7 +533,7 @@ function guesscolparsers(str::AbstractString, header, opts::LocalOpts, pos::Int,
 
     # override guesses with user request
     for (i, v) in optionsiter(colparsers, header)
-        guess[i] = tofield(v, opts)
+        guess[i] = tofield(v, opts, stringarraytype)
     end
     guess, pos
 end
@@ -662,7 +661,7 @@ end
 function quotedsplit(str, opts, includequotes, i=firstindex(str), l=lastindex(str))
     strtok = Quoted(StringToken(String), opts.quotechar, opts.escapechar, required=false,
                     includequotes=includequotes)
-                    
+
     f = Field(strtok, eoldelim=true)
     strs = String[]
     if l == 0

src/field.jl

@@ -146,10 +146,10 @@ const pre_comp_exp_double = Double64[Double64(10.0)^i for i=0:308]
     f = Float64(f1)
     r = f1 - Int64(f) # get the remainder
     x = Double64(f) + Double64(r)
-  
+
     maxexp = 308
     minexp = -256
-  
+
     if exp >= 0
         x *= pre_comp_exp_double[exp+1]
     else
@@ -211,7 +211,7 @@ end
     y3 = iterate(str, i)
     if y3!==nothing && _is_e(str, i)
         i = y3[2]
-    
+
         y4 = iterate(str, i)
         if y4!==nothing
             enegate = false
@@ -266,7 +266,7 @@ function tryparsenext(::Percentage, str, i, len, opts)
         # parse away the % char
         ii = eatwhitespaces(str, ii, len)
         y = iterate(str, ii)
-        if y===nothing 
+        if y===nothing
             return Nullable{Float64}(), ii # failed to parse %
         else
             c = y[1]; k = y[2]
@@ -295,6 +295,8 @@ show(io::IO, c::StringToken) = print(io, "<string>")
 fromtype(::Type{S}) where {S<:AbstractString} = StringToken(S)
 
 function tryparsenext(s::StringToken{T}, str, i, len, opts) where {T}
+    inside_quoted_strong = Char(opts.endchar) == Char(opts.quotechar)
+    escapecount = 0
     R = Nullable{T}
     p = ' '
     i0 = i
@@ -312,10 +314,15 @@ function tryparsenext(s::StringToken{T}, str, i, len, opts) where {T}
     y2 = iterate(str, i)
     while y2!==nothing
         c = y2[1]; ii = y2[2]
+
+        if inside_quoted_strong && p==Char(opts.escapechar)
+            escapecount += 1
+        end
+
         if opts.spacedelim && (c == ' ' || c == '\t')
             break
         elseif !opts.spacedelim && c == Char(opts.endchar)
-            if Char(opts.endchar) == Char(opts.quotechar)
+            if inside_quoted_strong
                 # this means we're inside a quoted string
                 if Char(opts.quotechar) == Char(opts.escapechar)
                     # sometimes the quotechar is the escapechar
@@ -358,14 +365,41 @@ function tryparsenext(s::StringToken{T}, str, i, len, opts) where {T}
         y2 = iterate(str, i)
     end
 
-    return R(_substring(T, str, i0, i-1)), i
+    return R(_substring(T, str, i0, i-1, escapecount, opts)), i
 end
 
-@inline function _substring(::Type{String}, str, i, j)
-    String(str[i:thisind(str, j)])
+@inline function _substring(::Type{String}, str, i, j, escapecount, opts)
+    if escapecount > 0
+        buf = IOBuffer(sizehint=j-i+1-escapecount)
+        cur_i = i
+        c = str[cur_i]
+        if opts.includequotes && c==Char(opts.quotechar)
+            print(buf, c)
+            cur_i = nextind(str, cur_i)
+        end
+        while cur_i <= j
+            c = str[cur_i]
+            if c == Char(opts.escapechar)
+                next_i = nextind(str, cur_i)
+                if next_i <= j && str[next_i] == Char(opts.quotechar)
+                    print(buf, str[next_i])
+                    cur_i = next_i
+                else
+                    print(buf, c)
+                end
+            else
+                print(buf, c)
+            end
+            cur_i = nextind(str, cur_i)
+        end
+        return String(take!(buf))
+    else
+        return unsafe_string(pointer(str, i), j-i+1)
+    end
 end
 
-@inline function _substring(::Type{T}, str, i, j) where {T<:SubString}
+@inline function _substring(::Type{T}, str, i, j, escapecount, opts) where {T<:SubString}
+    escapecount > 0 && error("Not yet handled.")
     T(str, i, thisind(j))
 end
 
@@ -375,11 +409,12 @@ fromtype(::Type{StrRange}) = StringToken(StrRange)
     unsafe_string(pointer(str, 1 + r.offset), r.length)
 end
 
-@inline function _substring(::Type{StrRange}, str, i, j)
-    StrRange(i - 1, j - i + 1)
+@inline function _substring(::Type{StrRange}, str, i, j, escapecount, opts)
+    StrRange(i - 1, j - i + 1, escapecount)
 end
 
-@inline function _substring(::Type{<:WeakRefString}, str, i, j)
+@inline function _substring(::Type{<:WeakRefString}, str, i, j, escapecount, opts)
+    escapecount > 0 && error("Not yet handled.")
     WeakRefString(convert(Ptr{UInt8}, pointer(str, i)), j - i + 1)
 end
 

src/guesstype.jl

@@ -52,7 +52,7 @@ function getquotechar(x)
     return '\0'
 end
 
-function guesstoken(x, opts, @nospecialize(prev_guess)=Unknown(), nastrings=NA_STRINGS)
+function guesstoken(x, opts, @nospecialize(prev_guess)=Unknown(), nastrings=NA_STRINGS, stringarraytype=StringArray)
     q = getquotechar(x)
 
     if isa(prev_guess, StringToken)
@@ -65,18 +65,18 @@ function guesstoken(x, opts, @nospecialize(prev_guess)=Unknown(), nastrings=NA_S
         else
             prev_inner = prev_guess
         end
-        inner_token = guesstoken(strip(strip(x, q)), opts, prev_inner, nastrings)
+        inner_token = guesstoken(strip(strip(x, q)), opts, prev_inner, nastrings, stringarraytype)
         return Quoted(inner_token, opts.quotechar, opts.escapechar)
     elseif isa(prev_guess, Quoted)
         # but this token is not quoted
-        return Quoted(guesstoken(x, opts, prev_guess.inner, nastrings), opts.quotechar, opts.escapechar)
+        return Quoted(guesstoken(x, opts, prev_guess.inner, nastrings, stringarraytype), opts.quotechar, opts.escapechar)
     elseif isa(prev_guess, NAToken)
         # This column is nullable
         if isna(x, nastrings)
             # x is null too, return previous guess
             return prev_guess
         else
-            tok = guesstoken(x, opts, prev_guess.inner, nastrings)
+            tok = guesstoken(x, opts, prev_guess.inner, nastrings, stringarraytype)
             if isa(tok, StringToken)
                 return tok # never wrap a string in NAToken
             elseif isa(tok, Quoted)
@@ -108,17 +108,17 @@ function guesstoken(x, opts, @nospecialize(prev_guess)=Unknown(), nastrings=NA_S
                 return Numeric(promote_type(T, fieldtype(prev_guess)))
             else
                 # something like a date turned into a single number?
-                return StringToken(StrRange)
+                return StringToken(stringarraytype<:StringArray ? StrRange : String)
             end
         else
             # fast-path
             if length(filter(isnumeric, x)) < 4
-                return StringToken(StrRange)
+                return StringToken(stringarraytype<:StringArray ? StrRange : String)
             end
 
             maybedate = guessdateformat(x)
             if maybedate === nothing
-                return StringToken(StrRange)
+                return StringToken(stringarraytype<:StringArray ? StrRange : String)
             else
                 return maybedate
             end

src/record.jl

@@ -95,12 +95,8 @@ end
     PARSE_SUCCESS
 end
 
-@inline function setcell!(col::Array{String,1}, i, val::StrRange, str)
-    col[i] = alloc_string(str, val)
-    PARSE_SUCCESS
-end
-
 @inline Base.@propagate_inbounds function setcell!(col::StringVector, i, val::StrRange, str)
+    # TODO Properly handle the val.escapecount>0 case
     col[i] = WeakRefString(pointer(str, val.offset + 1), val.length)
     PARSE_SUCCESS
 end

src/utf8optimizations.jl

@@ -317,4 +317,116 @@ function tryparsenext(q::Quoted{T,S,<:UInt8,<:UInt8}, str::Union{VectorBackedUTF
 
     @label error
     return Nullable{T}(), i
-end
+end
+
+@inline function isnewline(b::UInt8)
+    b == UInt8(10) || b == UInt8(13)
+end
+
+function tryparsenext(s::StringToken{T}, str::Union{VectorBackedUTF8String, String}, i, len, opts::LocalOpts{<:UInt8,<:UInt8,<:UInt8}) where {T}
+    len = ncodeunits(str)
+    inside_quoted_strong = opts.endchar == opts.quotechar
+    escapecount = 0
+    R = Nullable{T}
+    p = UInt8(0)
+    i0 = i
+    if opts.includequotes
+        if i<=len
+            @inbounds b = codeunit(str, i)
+            if b==opts.quotechar
+                # advance counter so that
+                # the while loop doesn't react to opening quote
+                i += 1
+            end
+        end
+    end
+
+    while i<=len
+        @inbounds b = codeunit(str, i)
+        ii = i + 1
+
+        if inside_quoted_strong && p==opts.escapechar
+            escapecount += 1
+        end
+
+        if opts.spacedelim && (b == UInt8(32) || b == UInt8(9)) # 32 = ' ' and 9 = '\t'
+            break
+        elseif !opts.spacedelim && b == opts.endchar
+            if inside_quoted_strong
+                # this means we're inside a quoted string
+                if opts.quotechar == opts.escapechar
+                    # sometimes the quotechar is the escapechar
+                    # in that case we need to see the next char
+                    if ii > len
+                        if opts.includequotes
+                            i=ii
+                        end
+                        break
+                    else
+                        @inbounds next_b = codeunit(str, ii)
+                        if next_b == opts.quotechar
+                            # the current character is escaping the
+                            # next one
+                            i = ii + 1 # skip next char as well
+                            p = next_b
+                            continue
+                        end
+                    end
+                elseif p == opts.escapechar
+                    # previous char escaped this one
+                    i = ii
+                    p = b
+                    continue
+                end
+            end
+            if opts.includequotes
+                i = ii
+            end
+            break
+        elseif (!opts.includenewlines && isnewline(b))
+            break
+        end
+        i = ii
+        p = b
+    end
+
+    return R(_substring(T, str, i0, i-1, escapecount, opts)), i
+end
+
+@inline function _substring(::Type{String}, str::Union{VectorBackedUTF8String, String}, i, j, escapecount, opts::LocalOpts{<:UInt8,<:UInt8,<:UInt8})
+    if escapecount > 0
+        buffer = Vector{UInt8}(undef, j-i+1-escapecount)
+        cur_i = i
+        cur_buffer_i = 1
+        @inbounds c = codeunit(str, cur_i)
+        if opts.includequotes && c==opts.quotechar
+            @inbounds buffer[cur_buffer_i] = c
+            cur_i += 1
+            cur_buffer_i += 1
+        end
+        while cur_i <= j
+            @inbounds c = codeunit(str, cur_i)
+            if c == opts.escapechar
+                next_i = cur_i + 1
+                if next_i <= j
+                    @inbounds next_c = codeunit(str, next_i)
+                    if next_c == opts.quotechar
+                        @inbounds buffer[cur_buffer_i] = next_c
+                        cur_buffer_i += 1
+                        cur_i = next_i
+                    end
+                else
+                    @inbounds buffer[cur_buffer_i] = c
+                    cur_buffer_i += 1
+                end
+            else
+                @inbounds buffer[cur_buffer_i] = c
+                cur_buffer_i += 1
+            end
+            cur_i += 1
+        end
+        return String(buffer)
+    else
+        return unsafe_string(pointer(str, i), j-i+1)
+    end
+end