Merge pull request #20203 from JuliaLang/sk/chompreadline

readline(s), eachline: make `chomp=true` a keyword arg

Author: StefanKarpinski

NEWS.md

@@ -46,6 +46,10 @@ Breaking changes
 
 This section lists changes that do not have deprecation warnings.
 
+  * `readline`, `readlines` and `eachline` return lines without line endings by default.
+    You *must* use `readline(s, chomp=false)`, etc. to get the old behavior where lines
+    returned include trailing end-of-line character(s). ([#19944])
+
   * `String`s no longer have a `.data` field (as part of a significant performance
     improvement). Use `Vector{UInt8}(str)` to access a string as a byte array.
     However, allocating the `Vector` object has overhead. You can also use

base/LineEdit.jl

@@ -211,7 +211,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
     seek(buf, 0)
     moreinput = true # add a blank line if there is a trailing newline on the last line
     while moreinput
-        l = readline(buf)
+        l = readline(buf, chomp=false)
         moreinput = endswith(l, "\n")
         # We need to deal with on-screen characters, so use strwidth to compute occupied columns
         llength = strwidth(l)
@@ -549,7 +549,7 @@ end
 function edit_kill_line(s::MIState)
     buf = buffer(s)
     pos = position(buf)
-    killbuf = readline(buf)
+    killbuf = readline(buf, chomp=false)
     if length(killbuf) > 1 && killbuf[end] == '\n'
         killbuf = killbuf[1:end-1]
         char_move_left(buf)

base/REPL.jl

@@ -205,7 +205,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef)
         interrupted = false
         while true
             try
-                line *= readline(repl.terminal)
+                line *= readline(repl.terminal, chomp=false)
             catch e
                 if isa(e,InterruptException)
                     try # raise the debugger if present
@@ -337,7 +337,7 @@ An editor may have converted tabs to spaces at line """
 
 function hist_getline(file)
     while !eof(file)
-        line = readline(file)
+        line = readline(file, chomp=false)
         isempty(line) && return line
         line[1] in "\r\n" || return line
     end
@@ -995,7 +995,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef)
         if have_color
             print(repl.stream, input_color(repl))
         end
-        line = readline(repl.stream)
+        line = readline(repl.stream, chomp=false)
         if !isempty(line)
             ast = Base.parse_input_line(line)
             if have_color

base/client.jl

@@ -216,7 +216,7 @@ parse_input_line(s::AbstractString) = parse_input_line(String(s))
 function parse_input_line(io::IO)
     s = ""
     while !eof(io)
-        s = s*readline(io)
+        s *= readline(io, chomp=false)
         e = parse_input_line(s)
         if !(isa(e,Expr) && e.head === :incomplete)
             return e

base/deprecated.jl

@@ -1770,4 +1770,7 @@ eval(LibGit2, quote
         repository(x)
     end
 end)
+
+@deprecate EachLine(stream, ondone) EachLine(stream, ondone=ondone)
+
 # End deprecations scheduled for 0.6

base/io.jl

@@ -170,24 +170,48 @@ The text is assumed to be encoded in UTF-8.
 readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename)
 
 """
-    readline(stream::IO=STDIN)
-    readline(filename::AbstractString)
+    readline(stream::IO=STDIN; chomp::Bool=true)
+    readline(filename::AbstractString; chomp::Bool=true)
+
+Read a single line of text from the given I/O stream or file (defaults to `STDIN`).
+When reading from a file, the text is assumed to be encoded in UTF-8. Lines in the
+input end with `'\\n'` or `"\\r\\n"` or the end of an input stream. When `chomp` is
+true (as it is by default), these trailing newline characters are removed from the
+line before it is returned. When `chomp` is false, they are returned as part of the
+line.
+"""
+function readline(filename::AbstractString; chomp::Bool=true)
+    open(filename) do f
+        readline(f, chomp=chomp)
+    end
+end
 
-Read a single line of text, including a trailing newline character (if one is reached before
-the end of the input), from the given I/O stream or file (defaults to `STDIN`).
-When reading from a file, the text is assumed to be encoded in UTF-8.
-"""
-readline(filename::AbstractString) = open(readline, filename)
+function readline(s::IO=STDIN; chomp::Bool=true)
+    line = readuntil(s, 0x0a)
+    i = length(line)
+    if !chomp || i == 0 || line[i] != 0x0a
+        return String(line)
+    elseif i < 2 || line[i-1] != 0x0d
+        return String(resize!(line,i-1))
+    else
+        return String(resize!(line,i-2))
+    end
+end
 
 """
-    readlines(stream::IO)
-    readlines(filename::AbstractString)
+    readlines(stream::IO=STDIN; chomp::Bool=true)
+    readlines(filename::AbstractString; chomp::Bool=true)
 
-Read all lines of an I/O stream or a file as a vector of strings.
-The text is assumed to be encoded in UTF-8.
+Read all lines of an I/O stream or a file as a vector of strings. Behavior is
+equivalent to saving the result of reading `readline` repeatedly with the same
+arguments and saving the resulting lines as a vector of strings.
 """
-readlines(filename::AbstractString) = open(readlines, filename)
-
+function readlines(filename::AbstractString; chomp::Bool=true)
+    open(filename) do f
+        readlines(f, chomp=chomp)
+    end
+end
+readlines(s::IO=STDIN; chomp::Bool=true) = collect(eachline(s, chomp=chomp))
 
 ## byte-order mark, ntoh & hton ##
 
@@ -454,9 +478,6 @@ function readuntil(s::IO, t::AbstractString)
     return String(take!(out))
 end
 
-readline() = readline(STDIN)
-readline(s::IO) = readuntil(s, '\n')
-
 """
     readchomp(x)
 
@@ -520,35 +541,39 @@ readstring(filename::AbstractString) = open(readstring, filename)
 type EachLine
     stream::IO
     ondone::Function
-    EachLine(stream) = EachLine(stream, ()->nothing)
-    EachLine(stream, ondone) = new(stream, ondone)
+    chomp::Bool
+
+    EachLine(stream::IO=STDIN; ondone::Function=()->nothing, chomp::Bool=true) =
+        new(stream, ondone, chomp)
 end
 
 """
-    eachline(stream::IO)
-    eachline(filename::AbstractString)
+    eachline(stream::IO=STDIN; chomp::Bool=true)
+    eachline(filename::AbstractString; chomp::Bool=true)
 
-Create an iterable object that will yield each line from an I/O stream or a file.
-The text is assumed to be encoded in UTF-8.
+Create an iterable `EachLine` object that will yield each line from an I/O stream
+or a file. Iteration calls `readline` on the stream argument repeatedly with
+`chomp` passed through, determining whether trailing end-of-line characters are
+removed. When called with a file name, the file is opened once at the beginning of
+iteration and closed at the end. If iteration is interrupted, the file will be
+closed when the `EachLine` object is garbage collected.
 """
-eachline(stream::IO) = EachLine(stream)
-function eachline(filename::AbstractString)
+eachline(stream::IO=STDIN; chomp::Bool=true) = EachLine(stream, chomp=chomp)
+
+function eachline(filename::AbstractString; chomp::Bool=true)
     s = open(filename)
-    EachLine(s, ()->close(s))
+    EachLine(s, ondone=()->close(s), chomp=chomp)
 end
 
 start(itr::EachLine) = nothing
-function done(itr::EachLine, nada)
-    if !eof(itr.stream)
-        return false
-    end
+function done(itr::EachLine, ::Void)
+    eof(itr.stream) || return false
     itr.ondone()
     true
 end
-next(itr::EachLine, nada) = (readline(itr.stream), nothing)
-eltype(::Type{EachLine}) = String
+next(itr::EachLine, ::Void) = (readline(itr.stream, chomp=itr.chomp), nothing)
 
-readlines(s=STDIN) = collect(eachline(s))
+eltype(::Type{EachLine}) = String
 
 iteratorsize(::Type{EachLine}) = SizeUnknown()
 

base/iostream.jl

@@ -222,16 +222,16 @@ take!(s::IOStream) =
     ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Void},), s.ios)
 
 function readuntil(s::IOStream, delim::UInt8)
-    ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 0)
+    ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0)
 end
 
 # like readuntil, above, but returns a String without requiring a copy
 function readuntil_string(s::IOStream, delim::UInt8)
-    ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 1)
+    ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 1, false)
 end
 
-function readline(s::IOStream)
-    ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1)
+function readline(s::IOStream; chomp::Bool=true)
+    ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp)
 end
 
 function readbytes_all!(s::IOStream, b::Array{UInt8}, nb)

base/libgit2/callbacks.jl

@@ -99,7 +99,8 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}},
         else
             # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED"
             open(privatekey) do f
-                passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED")
+                readline(f)
+                passphrase_required = readline(f) == "Proc-Type: 4,ENCRYPTED"
             end
         end
 

base/libgit2/utils.jl

@@ -31,7 +31,7 @@ function prompt(msg::AbstractString; default::AbstractString="", password::Bool=
         Base.getpass(msg)
     else
         print(msg)
-        chomp(readline(STDIN))
+        readline()
     end
     isempty(uinput) ? default : uinput
 end

base/markdown/Common/block.jl

@@ -61,7 +61,7 @@ function hashheader(stream::IO, md::MD)
             return false
 
         if c != '\n' # Empty header
-            h = readline(stream) |> strip
+            h = strip(readline(stream))
             h = match(r"(.*?)( +#+)?$", h).captures[1]
             buffer = IOBuffer()
             print(buffer, h)
@@ -76,11 +76,11 @@ end
 function setextheader(stream::IO, md::MD)
     withstream(stream) do
         eatindent(stream) || return false
-        header = readline(stream) |> strip
+        header = strip(readline(stream))
         header == "" && return false
 
         eatindent(stream) || return false
-        underline = readline(stream) |> strip
+        underline = strip(readline(stream))
         length(underline) < 3 && return false
         u = underline[1]
         u in "-=" || return false
@@ -108,7 +108,7 @@ function indentcode(stream::IO, block::MD)
         buffer = IOBuffer()
         while !eof(stream)
             if startswith(stream, "    ") || startswith(stream, "\t")
-                write(buffer, readline(stream))
+                write(buffer, readline(stream, chomp=false))
             elseif blankline(stream)
                 write(buffer, '\n')
             else
@@ -139,10 +139,10 @@ function footnote(stream::IO, block::MD)
         else
             ref = match(regex, str).captures[1]
             buffer = IOBuffer()
-            write(buffer, readline(stream))
+            write(buffer, readline(stream, chomp=false))
             while !eof(stream)
                 if startswith(stream, "    ")
-                    write(buffer, readline(stream))
+                    write(buffer, readline(stream, chomp=false))
                 elseif blankline(stream)
                     write(buffer, '\n')
                 else
@@ -174,7 +174,7 @@ function blockquote(stream::IO, block::MD)
         empty = true
         while eatindent(stream) && startswith(stream, '>')
             startswith(stream, " ")
-            write(buffer, readline(stream))
+            write(buffer, readline(stream, chomp=false))
             empty = false
         end
         empty && return false
@@ -229,7 +229,7 @@ function admonition(stream::IO, block::MD)
         buffer = IOBuffer()
         while !eof(stream)
             if startswith(stream, "    ")
-                write(buffer, readline(stream))
+                write(buffer, readline(stream, chomp=false))
             elseif blankline(stream)
                 write(buffer, '\n')
             else
@@ -305,7 +305,7 @@ function list(stream::IO, block::MD)
                 newline = false
                 if startswith(stream, " "^indent)
                     # Indented text that is part of the current list item.
-                    print(buffer, readline(stream))
+                    print(buffer, readline(stream, chomp=false))
                 else
                     matched = startswith(stream, regex)
                     if isempty(matched)
@@ -316,7 +316,7 @@ function list(stream::IO, block::MD)
                         # Start of a new list item.
                         count += 1
                         count > 1 && pushitem!(list, buffer)
-                        print(buffer, readline(stream))
+                        print(buffer, readline(stream, chomp=false))
                     end
                 end
             end

base/markdown/GitHub/GitHub.jl

@@ -30,7 +30,7 @@ function fencedcode(stream::IO, block::MD)
                     seek(stream, line_start)
                 end
             end
-            write(buffer, readline(stream))
+            write(buffer, readline(stream, chomp=false))
         end
         return false
     end

base/markdown/GitHub/table.jl

@@ -7,7 +7,7 @@ end
 
 function parserow(stream::IO)
     withstream(stream) do
-        line = readline(stream) |> chomp
+        line = readline(stream)
         row = split(line, r"(?<!\\)\|")
         length(row) == 1 && return
         row[1] == "" && shift!(row)

base/markdown/parse/util.jl

@@ -46,7 +46,7 @@ function linecontains(io::IO, chars; allow_whitespace = true,
                                      eat = true,
                                      allowempty = false)
     start = position(io)
-    l = readline(io) |> chomp
+    l = readline(io)
     length(l) == 0 && return allowempty
 
     result = allowempty
@@ -99,7 +99,7 @@ function startswith(stream::IO, r::Regex; eat = true, padding = false)
     @assert Base.startswith(r.pattern, "^")
     start = position(stream)
     padding && skipwhitespace(stream)
-    line = chomp(readline(stream))
+    line = readline(stream)
     seek(stream, start)
     m = match(r, line)
     m === nothing && return ""

base/multi.jl

@@ -1628,9 +1628,9 @@ function redirect_worker_output(ident, stream)
         if startswith(line, "\tFrom worker ")
             # STDOUT's of "additional" workers started from an initial worker on a host are not available
             # on the master directly - they are routed via the initial worker's STDOUT.
-            print(line)
+            println(line)
         else
-            print("\tFrom worker $(ident):\t$line")
+            println("\tFrom worker $(ident):\t$line")
         end
     end
 end

base/pkg/dir.jl

@@ -67,7 +67,7 @@ end
 
 function getmetabranch()
     try
-        chomp(readline(joinpath(path(),"META_BRANCH")))
+        readline(joinpath(path(),"META_BRANCH"))
     catch err
         META_BRANCH
     end

base/pkg/entry.jl

@@ -608,8 +608,7 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set())
         empty!(Base.DL_LOAD_PATH)
         append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH)))
         open("$(escape_string(errfile))", "a") do f
-            for path_ in eachline(STDIN)
-                path = chomp(path_)
+            for path in eachline(STDIN)
                 pkg = basename(dirname(dirname(path)))
                 try
                     info("Building \$pkg")

base/pkg/reqs.jl

@@ -66,7 +66,6 @@ end
 function read(readable::Union{IO,Base.AbstractCmd})
     lines = Line[]
     for line in eachline(readable)
-        line = chomp(line)
         push!(lines, ismatch(r"^\s*(?:#|$)", line) ? Comment(line) : Requirement(line))
     end
     return lines