Internal error parsing malformed expression

[maleadt]

I encountered the following internal error on 1.10:

julia> code_llvm(logpdf, Tuple{Distributions.MvNormal{Float64, PDMats.PDMat{Float64, Array{Float64, 2}}, Array{Float64, 1}}, Array{ForwardDiff.Dual{ForwardDiff.Tag{typeof(Main.logp), Float64}, Float64, 10}, 1}) from logpdf(Distributions.Distribution{Distributions.ArrayLikeVariate{N}, S} where S<:Distributions.ValueSupport, AbstractArray{var"#s12", N} where var"#s12"<:Real) where {N}})
┌ Error: JuliaSyntax parser failed — falling back to flisp!
│   exception =
│    Internal error: length(args) == 1
│    Stacktrace:
│      [1] error(::String, ::String)
│        @ Base ./error.jl:44
│      [2] internal_error(strs::String)
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/utils.jl:23
│      [3] _internal_node_to_Expr(source::Base.JuliaSyntax.SourceFile, srcrange::UnitRange{…}, head::Base.JuliaSyntax.SyntaxHead, childranges::Vector{…}, childheads::Vector{…}, args::Vector{…})
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/expr.jl:191
│      [4] _to_expr(node::Base.JuliaSyntax.SyntaxNode)
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/expr.jl:496
│      [5] _to_expr(node::Base.JuliaSyntax.SyntaxNode) (repeats 2 times)
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/expr.jl:495
│      [6] Expr(node::Base.JuliaSyntax.SyntaxNode)
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/expr.jl:500
│      [7] core_parser_hook(code::String, filename::String, lineno::Int64, offset::Int64, options::Symbol)
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/hooks.jl:209
│      [8] invoke_in_world(::UInt64, ::Any, ::Any, ::Vararg{Any}; kwargs::@Kwargs{})
│        @ Base ./essentials.jl:901
│      [9] invoke_in_world(::UInt64, ::Any, ::Any, ::Vararg{Any})
│        @ Base ./essentials.jl:898
│     [10] (::Base.JuliaSyntax.var"#invoke_fixedworld#120"{…})(::String, ::Vararg{…}; kws::@Kwargs{})
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/hooks.jl:118
│     [11] (::Base.JuliaSyntax.var"#invoke_fixedworld#120"{…})(::String, ::Vararg{…})
│        @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/hooks.jl:117
│     [12] _parse_string(text::String, filename::String, lineno::Int64, index::Int64, options::Symbol)
│        @ Base.Meta ./meta.jl:200
│     [13] parseall(text::String; filename::String, lineno::Int64)
│        @ Base.Meta ./meta.jl:294 [inlined]
│     [14] _parse_input_line_core(s::String, filename::String)
│        @ Base ./client.jl:174 [inlined]
│     [15] parse_input_line(s::String; filename::String, depwarn::Bool)
│        @ Base ./client.jl:192 [inlined]
│     [16] (::REPL.var"#93#103"{REPL.LineEditREPL, REPL.REPLHistoryProvider})(x::Any)
│        @ REPL ~/Julia/src/julia/build/dev/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:1064
│     [17] invokelatest(f::REPL.var"#93#103"{REPL.LineEditREPL, REPL.REPLHistoryProvider}, args::String; kwargs::@Kwargs{})
│        @ Base ./essentials.jl:867 [inlined]
│     [18] invokelatest(f::REPL.var"#93#103"{REPL.LineEditREPL, REPL.REPLHistoryProvider}, args::String)
│        @ Base ./essentials.jl:864 [inlined]
│     [19] (::REPL.var"#do_respond#80"{…})(s::REPL.LineEdit.MIState, buf::Any, ok::Bool)
│        @ REPL ~/Julia/src/julia/build/dev/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:890
│     [20] invokelatest(::Any, ::REPL.LineEdit.MIState, ::IOBuffer, ::Bool; kwargs::@Kwargs{})
│        @ Base ./essentials.jl:867 [inlined]
│     [21] invokelatest(::Any, ::REPL.LineEdit.MIState, ::IOBuffer, ::Bool)
│        @ Base ./essentials.jl:864 [inlined]
│     [22] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
│        @ REPL.LineEdit ~/Julia/src/julia/build/dev/usr/share/julia/stdlib/v1.10/REPL/src/LineEdit.jl:2656
│     [23] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
│        @ REPL ~/Julia/src/julia/build/dev/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:1297
│     [24] (::REPL.var"#62#68"{REPL.LineEditREPL, REPL.REPLBackendRef})()
│        @ REPL ~/Julia/src/julia/build/dev/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:373
│    Some type information was truncated. Use `show(err)` to see complete types.
│   offset = 0
│   code = "code_llvm(logpdf, Tuple{Distributions.MvNormal{Float64, PDMats.PDMat{Float64, Array{Float64, 2}}, Array{Float64, 1}}, Array{ForwardDiff.Dual{ForwardDiff.Tag{typeof(Main.logp), Float64}, Float64, 10}, 1}) from logpdf(Distributions.Distribution{Distributions.ArrayLikeVariate{N}, S} where S<:Distributions.ValueSupport, AbstractArray{var\"#s12\", N} where var\"#s12\"<:Real) where {N}})"
└ @ Base.JuliaSyntax ~/Julia/src/julia/base/JuliaSyntax/src/hooks.jl:258
ERROR: syntax: missing comma or } in argument list
Stacktrace:
 [1] top-level scope
   @ none:1

I'm not sure whether this still reproduces on latest JuliaSyntax.jl; maybe it'd be good to add a section to the README how to easily test the latest version of the parser to see if it's worth filing a bug?

[c42f]

This seems to be an issue with the hooks rather than the parser. But definitely a bug, and it's still current - thanks for reporting it ❤️

You can just do JuliaSyntax.enable_in_core!() to use the version you've got checked out as the default parser. I'll add that to the howto section in the docs I guess. Or maybe the readme... it's hard to know which parts to put where.

[c42f]

Minimal reproduction:

julia> JuliaSyntax.core_parser_hook("x var\"y\"", "foo.jl", 0, :all)
┌ Error: JuliaSyntax parser failed — falling back to flisp!
│   exception =
│    Internal error: length(args) == 1
│    Stacktrace:
│      [1] error(::String, ::String)
│        @ Base ./error.jl:44
│      [2] internal_error(strs::String)
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/utils.jl:23
│      [3] _internal_node_to_Expr(source::SourceFile, srcrange::UnitRange{Int64}, head::JuliaSyntax.SyntaxHead, childranges::Vector{UnitRange{Int64}}, childheads::Vector{JuliaSyntax.SyntaxHead}, args::Vector{Any})
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:200
│      [4] _to_expr(node::SyntaxNode)
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:524
│      [5] _to_expr(node::SyntaxNode) (repeats 2 times)
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:523
│      [6] Expr(node::SyntaxNode)
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:528
│      [7] core_parser_hook(code::String, filename::String, lineno::Int64, offset::Int64, options::Symbol)
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/hooks.jl:209
│      [8] core_parser_hook(code::String, filename::String, offset::Int64, options::Symbol)
│        @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/hooks.jl:271
│      [9] top-level scope
│        @ REPL[22]:1
│     [10] eval
│        @ ./boot.jl:370 [inlined]
│     [11] eval_user_input(ast::Any, backend::REPL.REPLBackend, mod::Module)
│        @ REPL ~/.julia/juliaup/julia-1.9.2+0.x64.linux.gnu/share/julia/stdlib/v1.9/REPL/src/REPL.jl:153
│     [12] repl_backend_loop(backend::REPL.REPLBackend, get_module::Function)
│        @ REPL ~/.julia/juliaup/julia-1.9.2+0.x64.linux.gnu/share/julia/stdlib/v1.9/REPL/src/REPL.jl:249
│     [13] start_repl_backend(backend::REPL.REPLBackend, consumer::Any; get_module::Function)
│        @ REPL ~/.julia/juliaup/julia-1.9.2+0.x64.linux.gnu/share/julia/stdlib/v1.9/REPL/src/REPL.jl:234
│     [14] run_repl(repl::REPL.AbstractREPL, consumer::Any; backend_on_current_task::Bool, backend::Any)
│        @ REPL ~/.julia/juliaup/julia-1.9.2+0.x64.linux.gnu/share/julia/stdlib/v1.9/REPL/src/REPL.jl:379
│     [15] run_repl(repl::REPL.AbstractREPL, consumer::Any)
│        @ REPL ~/.julia/juliaup/julia-1.9.2+0.x64.linux.gnu/share/julia/stdlib/v1.9/REPL/src/REPL.jl:365
│     [16] (::Base.var"#1017#1019"{Bool, Bool, Bool})(REPL::Module)
│        @ Base ./client.jl:421
│     [17] #invokelatest#2
│        @ ./essentials.jl:816 [inlined]
│     [18] invokelatest
│        @ ./essentials.jl:813 [inlined]
│     [19] run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_file::Bool, color_set::Bool)
│        @ Base ./client.jl:405
│     [20] exec_options(opts::Base.JLOptions)
│        @ Base ./client.jl:322
│     [21] _start()
│        @ Base ./client.jl:522
│   offset = 0
│   code = "x var\"y\""
└ @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/hooks.jl:258

[c42f]

Ok. I think what's going on here is that a node/token of type K"var" with zero children is bumped by the parser during recovery here

JuliaSyntax.jl/src/parser.jl

Line 488 in b941fc0

bump(ps)

When building the tree after parsing

• We can't tell the difference between token K"var" which is bump'ed vs an empty node K"var" - this is implicit (see here

  JuliaSyntax.jl/src/parse_stream.jl

  Line 1034 in b941fc0

  children = (is_syntax_kind(h) || is_keyword(h)) ?

  )
• A K"var" without any children breaks the expected invariant that these should have exactly one child.

This first point is pretty awkward and if it could be fixed that would be ideal.