diff --git a/NEWS.md b/NEWS.md index 5d37f11030bc7..c69c171fefeb8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -46,6 +46,10 @@ Breaking changes This section lists changes that do not have deprecation warnings. + * `readline`, `readlines` and `eachline` return lines without line endings by default. + You *must* use `readline(s, chomp=false)`, etc. to get the old behavior where lines + returned include trailing end-of-line character(s). ([#19944]) + * `String`s no longer have a `.data` field (as part of a significant performance improvement). Use `Vector{UInt8}(str)` to access a string as a byte array. However, allocating the `Vector` object has overhead. You can also use diff --git a/base/LineEdit.jl b/base/LineEdit.jl index a6bf2fc009719..3a952c4bd193b 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -211,7 +211,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf seek(buf, 0) moreinput = true # add a blank line if there is a trailing newline on the last line while moreinput - l = readline(buf) + l = readline(buf, chomp=false) moreinput = endswith(l, "\n") # We need to deal with on-screen characters, so use strwidth to compute occupied columns llength = strwidth(l) @@ -549,7 +549,7 @@ end function edit_kill_line(s::MIState) buf = buffer(s) pos = position(buf) - killbuf = readline(buf) + killbuf = readline(buf, chomp=false) if length(killbuf) > 1 && killbuf[end] == '\n' killbuf = killbuf[1:end-1] char_move_left(buf) diff --git a/base/REPL.jl b/base/REPL.jl index 8f21e231c31cb..928a4cd8ce612 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -205,7 +205,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef) interrupted = false while true try - line *= readline(repl.terminal) + line *= readline(repl.terminal, chomp=false) catch e if isa(e,InterruptException) try # raise the debugger if present @@ -337,7 +337,7 @@ An editor may have converted tabs to spaces at line """ function hist_getline(file) while !eof(file) - line = readline(file) + line = readline(file, chomp=false) isempty(line) && return line line[1] in "\r\n" || return line end @@ -995,7 +995,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef) if have_color print(repl.stream, input_color(repl)) end - line = readline(repl.stream) + line = readline(repl.stream, chomp=false) if !isempty(line) ast = Base.parse_input_line(line) if have_color diff --git a/base/client.jl b/base/client.jl index 9b7a8e9b71c0d..0221d221dba98 100644 --- a/base/client.jl +++ b/base/client.jl @@ -216,7 +216,7 @@ parse_input_line(s::AbstractString) = parse_input_line(String(s)) function parse_input_line(io::IO) s = "" while !eof(io) - s = s*readline(io) + s *= readline(io, chomp=false) e = parse_input_line(s) if !(isa(e,Expr) && e.head === :incomplete) return e diff --git a/base/deprecated.jl b/base/deprecated.jl index 6cbb1669286ab..3f8a9f947a541 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1770,4 +1770,7 @@ eval(LibGit2, quote repository(x) end end) + +@deprecate EachLine(stream, ondone) EachLine(stream, ondone=ondone) + # End deprecations scheduled for 0.6 diff --git a/base/io.jl b/base/io.jl index b04597cc2e2f1..562154e7df52a 100644 --- a/base/io.jl +++ b/base/io.jl @@ -170,24 +170,48 @@ The text is assumed to be encoded in UTF-8. readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename) """ - readline(stream::IO=STDIN) - readline(filename::AbstractString) + readline(stream::IO=STDIN; chomp::Bool=true) + readline(filename::AbstractString; chomp::Bool=true) + +Read a single line of text from the given I/O stream or file (defaults to `STDIN`). +When reading from a file, the text is assumed to be encoded in UTF-8. Lines in the +input end with `'\\n'` or `"\\r\\n"` or the end of an input stream. When `chomp` is +true (as it is by default), these trailing newline characters are removed from the +line before it is returned. When `chomp` is false, they are returned as part of the +line. +""" +function readline(filename::AbstractString; chomp::Bool=true) + open(filename) do f + readline(f, chomp=chomp) + end +end -Read a single line of text, including a trailing newline character (if one is reached before -the end of the input), from the given I/O stream or file (defaults to `STDIN`). -When reading from a file, the text is assumed to be encoded in UTF-8. -""" -readline(filename::AbstractString) = open(readline, filename) +function readline(s::IO=STDIN; chomp::Bool=true) + line = readuntil(s, 0x0a) + i = length(line) + if !chomp || i == 0 || line[i] != 0x0a + return String(line) + elseif i < 2 || line[i-1] != 0x0d + return String(resize!(line,i-1)) + else + return String(resize!(line,i-2)) + end +end """ - readlines(stream::IO) - readlines(filename::AbstractString) + readlines(stream::IO=STDIN; chomp::Bool=true) + readlines(filename::AbstractString; chomp::Bool=true) -Read all lines of an I/O stream or a file as a vector of strings. -The text is assumed to be encoded in UTF-8. +Read all lines of an I/O stream or a file as a vector of strings. Behavior is +equivalent to saving the result of reading `readline` repeatedly with the same +arguments and saving the resulting lines as a vector of strings. """ -readlines(filename::AbstractString) = open(readlines, filename) - +function readlines(filename::AbstractString; chomp::Bool=true) + open(filename) do f + readlines(f, chomp=chomp) + end +end +readlines(s::IO=STDIN; chomp::Bool=true) = collect(eachline(s, chomp=chomp)) ## byte-order mark, ntoh & hton ## @@ -454,9 +478,6 @@ function readuntil(s::IO, t::AbstractString) return String(take!(out)) end -readline() = readline(STDIN) -readline(s::IO) = readuntil(s, '\n') - """ readchomp(x) @@ -520,35 +541,39 @@ readstring(filename::AbstractString) = open(readstring, filename) type EachLine stream::IO ondone::Function - EachLine(stream) = EachLine(stream, ()->nothing) - EachLine(stream, ondone) = new(stream, ondone) + chomp::Bool + + EachLine(stream::IO=STDIN; ondone::Function=()->nothing, chomp::Bool=true) = + new(stream, ondone, chomp) end """ - eachline(stream::IO) - eachline(filename::AbstractString) + eachline(stream::IO=STDIN; chomp::Bool=true) + eachline(filename::AbstractString; chomp::Bool=true) -Create an iterable object that will yield each line from an I/O stream or a file. -The text is assumed to be encoded in UTF-8. +Create an iterable `EachLine` object that will yield each line from an I/O stream +or a file. Iteration calls `readline` on the stream argument repeatedly with +`chomp` passed through, determining whether trailing end-of-line characters are +removed. When called with a file name, the file is opened once at the beginning of +iteration and closed at the end. If iteration is interrupted, the file will be +closed when the `EachLine` object is garbage collected. """ -eachline(stream::IO) = EachLine(stream) -function eachline(filename::AbstractString) +eachline(stream::IO=STDIN; chomp::Bool=true) = EachLine(stream, chomp=chomp) + +function eachline(filename::AbstractString; chomp::Bool=true) s = open(filename) - EachLine(s, ()->close(s)) + EachLine(s, ondone=()->close(s), chomp=chomp) end start(itr::EachLine) = nothing -function done(itr::EachLine, nada) - if !eof(itr.stream) - return false - end +function done(itr::EachLine, ::Void) + eof(itr.stream) || return false itr.ondone() true end -next(itr::EachLine, nada) = (readline(itr.stream), nothing) -eltype(::Type{EachLine}) = String +next(itr::EachLine, ::Void) = (readline(itr.stream, chomp=itr.chomp), nothing) -readlines(s=STDIN) = collect(eachline(s)) +eltype(::Type{EachLine}) = String iteratorsize(::Type{EachLine}) = SizeUnknown() diff --git a/base/iostream.jl b/base/iostream.jl index 018aa0dd871f9..a2050119b78fb 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -222,16 +222,16 @@ take!(s::IOStream) = ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Void},), s.ios) function readuntil(s::IOStream, delim::UInt8) - ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 0) + ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0) end # like readuntil, above, but returns a String without requiring a copy function readuntil_string(s::IOStream, delim::UInt8) - ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 1) + ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 1, false) end -function readline(s::IOStream) - ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1) +function readline(s::IOStream; chomp::Bool=true) + ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp) end function readbytes_all!(s::IOStream, b::Array{UInt8}, nb) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 5c3543606e7c2..a38c31460a335 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -99,7 +99,8 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, else # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" open(privatekey) do f - passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + readline(f) + passphrase_required = readline(f) == "Proc-Type: 4,ENCRYPTED" end end diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index 9b0f0991d2c2a..31ccfbf6ff93e 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -31,7 +31,7 @@ function prompt(msg::AbstractString; default::AbstractString="", password::Bool= Base.getpass(msg) else print(msg) - chomp(readline(STDIN)) + readline() end isempty(uinput) ? default : uinput end diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index edc773082722b..9a8aed8de78f6 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -61,7 +61,7 @@ function hashheader(stream::IO, md::MD) return false if c != '\n' # Empty header - h = readline(stream) |> strip + h = strip(readline(stream)) h = match(r"(.*?)( +#+)?$", h).captures[1] buffer = IOBuffer() print(buffer, h) @@ -76,11 +76,11 @@ end function setextheader(stream::IO, md::MD) withstream(stream) do eatindent(stream) || return false - header = readline(stream) |> strip + header = strip(readline(stream)) header == "" && return false eatindent(stream) || return false - underline = readline(stream) |> strip + underline = strip(readline(stream)) length(underline) < 3 && return false u = underline[1] u in "-=" || return false @@ -108,7 +108,7 @@ function indentcode(stream::IO, block::MD) buffer = IOBuffer() while !eof(stream) if startswith(stream, " ") || startswith(stream, "\t") - write(buffer, readline(stream)) + write(buffer, readline(stream, chomp=false)) elseif blankline(stream) write(buffer, '\n') else @@ -139,10 +139,10 @@ function footnote(stream::IO, block::MD) else ref = match(regex, str).captures[1] buffer = IOBuffer() - write(buffer, readline(stream)) + write(buffer, readline(stream, chomp=false)) while !eof(stream) if startswith(stream, " ") - write(buffer, readline(stream)) + write(buffer, readline(stream, chomp=false)) elseif blankline(stream) write(buffer, '\n') else @@ -174,7 +174,7 @@ function blockquote(stream::IO, block::MD) empty = true while eatindent(stream) && startswith(stream, '>') startswith(stream, " ") - write(buffer, readline(stream)) + write(buffer, readline(stream, chomp=false)) empty = false end empty && return false @@ -229,7 +229,7 @@ function admonition(stream::IO, block::MD) buffer = IOBuffer() while !eof(stream) if startswith(stream, " ") - write(buffer, readline(stream)) + write(buffer, readline(stream, chomp=false)) elseif blankline(stream) write(buffer, '\n') else @@ -305,7 +305,7 @@ function list(stream::IO, block::MD) newline = false if startswith(stream, " "^indent) # Indented text that is part of the current list item. - print(buffer, readline(stream)) + print(buffer, readline(stream, chomp=false)) else matched = startswith(stream, regex) if isempty(matched) @@ -316,7 +316,7 @@ function list(stream::IO, block::MD) # Start of a new list item. count += 1 count > 1 && pushitem!(list, buffer) - print(buffer, readline(stream)) + print(buffer, readline(stream, chomp=false)) end end end diff --git a/base/markdown/GitHub/GitHub.jl b/base/markdown/GitHub/GitHub.jl index 3692a8c515971..e29fdf62cad6b 100644 --- a/base/markdown/GitHub/GitHub.jl +++ b/base/markdown/GitHub/GitHub.jl @@ -30,7 +30,7 @@ function fencedcode(stream::IO, block::MD) seek(stream, line_start) end end - write(buffer, readline(stream)) + write(buffer, readline(stream, chomp=false)) end return false end diff --git a/base/markdown/GitHub/table.jl b/base/markdown/GitHub/table.jl index b310742c94068..04485669dc936 100644 --- a/base/markdown/GitHub/table.jl +++ b/base/markdown/GitHub/table.jl @@ -7,7 +7,7 @@ end function parserow(stream::IO) withstream(stream) do - line = readline(stream) |> chomp + line = readline(stream) row = split(line, r"(? chomp + l = readline(io) length(l) == 0 && return allowempty result = allowempty @@ -99,7 +99,7 @@ function startswith(stream::IO, r::Regex; eat = true, padding = false) @assert Base.startswith(r.pattern, "^") start = position(stream) padding && skipwhitespace(stream) - line = chomp(readline(stream)) + line = readline(stream) seek(stream, start) m = match(r, line) m === nothing && return "" diff --git a/base/multi.jl b/base/multi.jl index b60019ee1c5b7..7b2af23da729e 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1628,9 +1628,9 @@ function redirect_worker_output(ident, stream) if startswith(line, "\tFrom worker ") # STDOUT's of "additional" workers started from an initial worker on a host are not available # on the master directly - they are routed via the initial worker's STDOUT. - print(line) + println(line) else - print("\tFrom worker $(ident):\t$line") + println("\tFrom worker $(ident):\t$line") end end end diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 78b9fea5e001b..3b4d2da1bbd9c 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -67,7 +67,7 @@ end function getmetabranch() try - chomp(readline(joinpath(path(),"META_BRANCH"))) + readline(joinpath(path(),"META_BRANCH")) catch err META_BRANCH end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 529cfdd44d83c..4701571e21bd7 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -608,8 +608,7 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH))) open("$(escape_string(errfile))", "a") do f - for path_ in eachline(STDIN) - path = chomp(path_) + for path in eachline(STDIN) pkg = basename(dirname(dirname(path))) try info("Building \$pkg") diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index 461e15b0abaf6..003b82496e3c9 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -66,7 +66,6 @@ end function read(readable::Union{IO,Base.AbstractCmd}) lines = Line[] for line in eachline(readable) - line = chomp(line) push!(lines, ismatch(r"^\s*(?:#|$)", line) ? Comment(line) : Requirement(line)) end return lines diff --git a/base/precompile.jl b/base/precompile.jl index c8ce400142c17..60f0878609a39 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -323,6 +323,8 @@ precompile(Base.read, (IOBuffer, Type{Char})) precompile(Base.read, (IOBuffer, Type{UInt8})) precompile(Base.read, (IOStream, Array{UInt32,1})) precompile(Base.readline, (String,)) +precompile(Base.readline, (IOBuffer,)) +precompile(Base.readline, (IOStream,)) precompile(Base.readuntil, (IOBuffer, Char)) precompile(Base.readuntil, (IOBuffer, UInt8)) precompile(Base.rehash!, (Dict{Any,Any}, Int)) diff --git a/base/process.jl b/base/process.jl index 26ef5247ebccc..875179d206a24 100644 --- a/base/process.jl +++ b/base/process.jl @@ -555,15 +555,16 @@ spawn_opts_inherit(in::Redirectable=RawFD(0), out::Redirectable=RawFD(1), err::R spawn(cmds::AbstractCmd, args...; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) = spawn(cmds, spawn_opts_swallow(args...)...; chain=chain) -function eachline(cmd::AbstractCmd, stdin) +function eachline(cmd::AbstractCmd, stdin; chomp::Bool=true) stdout = Pipe() processes = spawn(cmd, (stdin,stdout,STDERR)) close(stdout.in) out = stdout.out # implicitly close after reading lines, since we opened - return EachLine(out, ()->(close(out); success(processes) || pipeline_error(processes))) + return EachLine(out, chomp=chomp, + ondone=()->(close(out); success(processes) || pipeline_error(processes))) end -eachline(cmd::AbstractCmd) = eachline(cmd, DevNull) +eachline(cmd::AbstractCmd; chomp::Bool=true) = eachline(cmd, DevNull, chomp=chomp) # return a Process object to read-to/write-from the pipeline """ diff --git a/contrib/add_license_to_files.jl b/contrib/add_license_to_files.jl index 63314844937af..7cfaae2ab9f0a 100644 --- a/contrib/add_license_to_files.jl +++ b/contrib/add_license_to_files.jl @@ -135,7 +135,7 @@ function add_license_line!(unprocessed::Vector, src::AbstractString, new_license if ext in keys(ext_prefix) prefix = ext_prefix[ext] f = open(path, "r") - lines = readlines(f) + lines = readlines(f, chomp=false) close(f) isempty(lines) && (push!(unprocessed, path); continue) isempty(old_license) || check_lines!(path, lines, old_license, prefix, true) diff --git a/examples/clustermanager/0mq/ZMQCM.jl b/examples/clustermanager/0mq/ZMQCM.jl index 1f305939b2ca5..1fd7fad73c0df 100644 --- a/examples/clustermanager/0mq/ZMQCM.jl +++ b/examples/clustermanager/0mq/ZMQCM.jl @@ -271,7 +271,7 @@ end function print_worker_stdout(io, pid) @schedule while !eof(io) line = readline(io) - print("\tFrom worker $(pid):\t$line") + println("\tFrom worker $(pid):\t$line") end end diff --git a/examples/clustermanager/simple/UnixDomainCM.jl b/examples/clustermanager/simple/UnixDomainCM.jl index b9519cde60caa..0822997d70149 100644 --- a/examples/clustermanager/simple/UnixDomainCM.jl +++ b/examples/clustermanager/simple/UnixDomainCM.jl @@ -82,7 +82,7 @@ end function print_worker_stdout(io, pid) @schedule while !eof(io) line = readline(io) - print("\tFrom worker $(pid):\t$line") + println("\tFrom worker $(pid):\t$line") end end diff --git a/src/flisp/iostream.c b/src/flisp/iostream.c index 197e60be731d5..a8de001ad8532 100644 --- a/src/flisp/iostream.c +++ b/src/flisp/iostream.c @@ -333,7 +333,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_setbuf(&dest, data, 80, 0); char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil"); ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil"); - size_t n = ios_copyuntil(&dest, src, delim); + size_t n = ios_copyuntil(&dest, src, delim, 0); cv->len = n; if (dest.buf != data) { // outgrew initial space @@ -352,7 +352,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil"); ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil"); char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil"); - return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim)); + return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 0)); } value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) diff --git a/src/gen_sysimg_symtab.jl b/src/gen_sysimg_symtab.jl index bc8d79f516340..193aa947ee8c0 100644 --- a/src/gen_sysimg_symtab.jl +++ b/src/gen_sysimg_symtab.jl @@ -17,7 +17,7 @@ io,_ = open(pipeline(`strings -n 3 $fname`, `head -n 315`)) # 63 + 252 function outputline(io, line) - row = split(chomp(line), " ", keep=false) + row = split(line, " ", keep=false) println(io, "jl_symbol(\"", row[2], "\"),") end diff --git a/src/support/ios.c b/src/support/ios.c index 650a8690c0888..32cba8fceb618 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -802,7 +802,7 @@ size_t ios_copyall(ios_t *to, ios_t *from) #define LINE_CHUNK_SIZE 160 -size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) { size_t total = 0, avail = (size_t)(from->size - from->bpos); while (!ios_eof(from)) { @@ -821,7 +821,11 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) } else { size_t ntowrite = pd - (from->buf+from->bpos) + 1; - written = ios_write(to, from->buf+from->bpos, ntowrite); + size_t nchomp = 0; + if (chomp) { + nchomp = ios_nchomp(from, ntowrite); + } + written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp); from->bpos += ntowrite; total += written; return total; @@ -831,6 +835,19 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) return total; } +size_t ios_nchomp(ios_t *from, size_t ntowrite) +{ + assert(ntowrite > 0); + size_t nchomp; + if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { + nchomp = 2; + } + else { + nchomp = 1; + } + return nchomp; +} + static void _ios_init(ios_t *s) { // put all fields in a sane initial state @@ -1135,7 +1152,7 @@ char *ios_readline(ios_t *s) { ios_t dest; ios_mem(&dest, 0); - ios_copyuntil(&dest, s, '\n'); + ios_copyuntil(&dest, s, '\n', 0); size_t n; return ios_take_buffer(&dest, &n); } diff --git a/src/support/ios.h b/src/support/ios.h index 6acd52fb01838..02480237a3389 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -94,7 +94,8 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s); JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); -JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim); +JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp); +JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index 3976b80dc4451..e391c469de3fb 100644 --- a/src/sys.c +++ b/src/sys.c @@ -250,7 +250,7 @@ JL_DLLEXPORT jl_array_t *jl_take_buffer(ios_t *s) return a; } -JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) +JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint8_t chomp) { jl_array_t *a; // manually inlined common case @@ -258,7 +258,11 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) if (pd) { size_t n = pd-(s->buf+s->bpos)+1; if (str) { - jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, n); + size_t nchomp = 0; + if (chomp) { + nchomp = ios_nchomp(s, n); + } + jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, n - nchomp); s->bpos += n; return str; } @@ -271,7 +275,7 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) ios_t dest; ios_mem(&dest, 0); ios_setbuf(&dest, (char*)a->data, 80, 0); - size_t n = ios_copyuntil(&dest, s, delim); + size_t n = ios_copyuntil(&dest, s, delim, chomp); if (dest.buf != a->data) { a = jl_take_buffer(&dest); } diff --git a/test/file.jl b/test/file.jl index bf1af867fcb6b..5fdcda60f1505 100644 --- a/test/file.jl +++ b/test/file.jl @@ -285,7 +285,7 @@ reset(s) str = readline(s) @test startswith(str, "Marked!") mark(s) -@test readline(s) == "Hello world!\n" +@test readline(s) == "Hello world!" @test ismarked(s) unmark(s) @test !ismarked(s) @@ -335,7 +335,7 @@ end emptyfile = joinpath(dir, "empty") touch(emptyfile) emptyf = open(emptyfile) -@test isempty(readlines(emptyf)) +@test isempty(readlines(emptyf, chomp=false)) close(emptyf) rm(emptyfile) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 5d429a39317a2..995ca6ecfce22 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -49,7 +49,7 @@ end let io = IOBuffer("hamster\nguinea pig\nturtle") @test position(io) == 0 -@test readline(io) == "hamster\n" +@test readline(io) == "hamster" @test readstring(io) == "guinea pig\nturtle" @test_throws EOFError read(io,UInt8) seek(io,0) @@ -66,14 +66,28 @@ let io = PipeBuffer() @test_throws EOFError read(io,UInt8) @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 @test position(io) == 0 -@test readline(io) == "pancakes\n" +@test readline(io) == "pancakes" Base.compact(io) -@test readline(io) == "waffles\n" +@test readline(io) == "waffles" @test write(io,"whipped cream\n") > 0 -@test readline(io) == "blueberries\n" +@test readline(io) == "blueberries" @test_throws ArgumentError seek(io,0) @test_throws ArgumentError truncate(io,0) -@test readline(io) == "whipped cream\n" +@test readline(io) == "whipped cream" +@test write(io,"pancakes\nwaffles\nblueberries\n") > 0 +@test readlines(io) == String["pancakes", "waffles", "blueberries"] +write(io,"\n\r\n\n\r \n") > 0 +@test readlines(io, chomp=false) == String["\n", "\r\n", "\n", "\r \n"] +write(io,"\n\r\n\n\r \n") > 0 +@test readlines(io, chomp=true) == String["", "", "", "\r "] +@test write(io,"α\nβ\nγ\nδ") > 0 +@test readlines(io, chomp=false) == String["α\n","β\n","γ\n","δ"] +@test write(io,"α\nβ\nγ\nδ") > 0 +@test readlines(io, chomp=true) == String["α", "β", "γ", "δ"] +@test readlines(IOBuffer(""), chomp=false) == [] +@test readlines(IOBuffer(""), chomp=true) == [] +@test readlines(IOBuffer("first\nsecond"), chomp=false) == String["first\n", "second"] +@test readlines(IOBuffer("first\nsecond"), chomp=true) == String["first", "second"] Base.compact(io) @test position(io) == 0 @test ioslength(io) == 0 @@ -133,7 +147,7 @@ a = Array{UInt8}(1024) @test eof(io) end -@test isempty(readlines(IOBuffer())) +@test isempty(readlines(IOBuffer(), chomp=false)) # issue #8193 let io=IOBuffer("asdf") diff --git a/test/iterators.jl b/test/iterators.jl index 154e2efb93da9..dd76ef64edc4b 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -25,12 +25,23 @@ end # typed `collect` @test collect(Float64, Iterators.filter(isodd, [1,2,3,4]))[1] === 1.0 +# check direct EachLine constructor +let b = IOBuffer("foo\n") + @test collect(EachLine(b)) == ["foo"] + seek(b, 0) + @test collect(EachLine(b, chomp=false)) == ["foo\n"] + seek(b, 0) + @test collect(EachLine(b, ondone=()->0)) == ["foo"] + seek(b, 0) + @test collect(EachLine(b, chomp=false, ondone=()->0)) == ["foo\n"] +end + # enumerate (issue #6284) let b = IOBuffer("1\n2\n3\n"), a = [] for (i,x) in enumerate(eachline(b)) push!(a, (i,x)) end - @test a == [(1,"1\n"),(2,"2\n"),(3,"3\n")] + @test a == [(1,"1"),(2,"2"),(3,"3")] end # zip eachline (issue #7369) diff --git a/test/perf/kernel/perf.jl b/test/perf/kernel/perf.jl index b5e50514288cc..1b6b30a888f5c 100644 --- a/test/perf/kernel/perf.jl +++ b/test/perf/kernel/perf.jl @@ -105,7 +105,7 @@ d = randn(len) writecsv("random.csv", rand(100000,4)) function parsecsv() - for line in EachLine(open("random.csv")) + for line in eachline("random.csv") split(line, ',') end end diff --git a/test/perf/shootout/k_nucleotide.jl b/test/perf/shootout/k_nucleotide.jl index 97f39624ff99d..d1353371ddcb3 100644 --- a/test/perf/shootout/k_nucleotide.jl +++ b/test/perf/shootout/k_nucleotide.jl @@ -62,12 +62,8 @@ end function k_nucleotide(infile="knucleotide-input.txt") input = open(infile, "r") - three = ">THREE " - while true - line = readline(input) - if length(line) >= length(three) && line[1:length(three)] == three - break - end + for line in eachline(input) + startswith(line, ">THREE ") && break end data = collect(readstring(input)) # delete the newlines and convert to upper case diff --git a/test/read.jl b/test/read.jl index 42bce748e9538..ba54963ef1499 100644 --- a/test/read.jl +++ b/test/read.jl @@ -241,12 +241,16 @@ for (name, f) in l cleanup() verbose && println("$name readline...") - @test readline(io()) == readline(IOBuffer(text)) - @test readline(io()) == readline(filename) + @test readline(io(), chomp=false) == readline(IOBuffer(text), chomp=false) + @test readline(io(), chomp=false) == readline(filename, chomp=false) verbose && println("$name readlines...") + @test readlines(io(), chomp=false) == readlines(IOBuffer(text), chomp=false) + @test readlines(io(), chomp=false) == readlines(filename, chomp=false) @test readlines(io()) == readlines(IOBuffer(text)) @test readlines(io()) == readlines(filename) + @test collect(eachline(io(), chomp=false)) == collect(eachline(IOBuffer(text), chomp=false)) + @test collect(eachline(io(), chomp=false)) == collect(eachline(filename, chomp=false)) @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) @test collect(eachline(io())) == collect(eachline(filename)) diff --git a/test/spawn.jl b/test/spawn.jl index 0f2f715ad315d..c09b29d0f47cd 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -182,18 +182,18 @@ let r, t, sock sock = connect(fetch(r)) mark(sock) @test ismarked(sock) - @test readline(sock) == "Hello, world!\n" - @test readline(sock) == "Goodbye, world...\n" + @test readline(sock) == "Hello, world!" + @test readline(sock) == "Goodbye, world..." @test reset(sock) == 0 @test !ismarked(sock) mark(sock) @test ismarked(sock) - @test readline(sock) == "Hello, world!\n" + @test readline(sock) == "Hello, world!" unmark(sock) @test !ismarked(sock) @test_throws ArgumentError reset(sock) @test !unmark(sock) - @test readline(sock) == "Goodbye, world...\n" + @test readline(sock) == "Goodbye, world..." #@test eof(sock) ## doesn't work close(sock) @test wait(t) @@ -326,8 +326,8 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test outfd != Base._fd(out.out) == Base.INVALID_OS_HANDLE @test nb_available(out) == 0 @test c == UInt8['w'] - @test lstrip(ln2) == "1\thello\n" - @test ln1 == "orld\n" + @test lstrip(ln2) == "1\thello" + @test ln1 == "orld" @test isempty(read(out)) @test eof(out) @test desc == "Pipe($infd open => $outfd active, 0 bytes waiting)"