From 39b83ce33b84a3f93be5021180cf55fff0b9c6c3 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Fri, 11 Oct 2024 20:08:11 +0900 Subject: [PATCH] make `CallInfo` propagate the `edges` list of `CodeInstance`s Remaining TODOs: - Finalize the format for `sv.edges`. There might be cases where no `edge::CodeInstance` exists as a result of `abstract_call_method`, and in such cases, we might still need to use `MethodInstance` in the `edges` list. - Ensure that when the local caching mode is specified (i.e. for const-prop'ed calls and call-site-inlined calls), the const-propped edge should be propagated instead of the regular edge. - Make use of the `CodeInstance` held by `CallInfo` during inlining for slightly better performance by avoiding the global cache lookup. --- base/compiler/abstractinterpretation.jl | 108 +++++++++++----------- base/compiler/ssair/inlining.jl | 13 +-- base/compiler/stmtinfo.jl | 96 +++++++++++--------- base/compiler/tfuncs.jl | 18 +++- base/compiler/typeinfer.jl | 115 +++++++++++------------- base/compiler/utilities.jl | 2 - stdlib/REPL/src/REPLCompletions.jl | 4 +- 7 files changed, 189 insertions(+), 167 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index d8e62bbdceb67..48411977acda3 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -72,7 +72,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), napplicable = length(applicable) multiple_matches = napplicable > 1 while i <= napplicable - match = applicable[i]::MethodMatch + (; match, edges, edge_idx) = applicable[i] method = match.method sig = match.spec_types if bail_out_toplevel_call(interp, InferenceLoopState(sig, rettype, all_effects), sv) @@ -94,7 +94,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), #end mresult = abstract_call_method(interp, method, sig, match.sparams, multiple_matches, si, sv)::Future function handle1(interp, sv) - local (; rt, exct, effects, volatile_inf_result) = mresult[] + local (; rt, exct, effects, edge, volatile_inf_result) = mresult[] this_conditional = ignorelimited(rt) this_rt = widenwrappedconditional(rt) this_exct = exct @@ -106,6 +106,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), mresult[], f.contents, this_arginfo, si, match, sv) const_result = volatile_inf_result if const_call_result !== nothing + # TODO override the edge with the const-prop' edge this_const_conditional = ignorelimited(const_call_result.rt) this_const_rt = widenwrappedconditional(const_call_result.rt) if this_const_rt βŠ‘β‚š this_rt @@ -163,6 +164,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), conditionals[2][i] = conditionals[2][i] βŠ”α΅’ cnd.elsetype end end + edges[edge_idx] = edge if i < napplicable && bail_out_call(interp, InferenceLoopState(sig, rettype, all_effects), sv) add_remark!(interp, sv, "Call inference reached maximally imprecise information. Bailing on.") seenall = false @@ -170,7 +172,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), end i += 1 return true - end + end # function handle1 if isready(mresult) && handle1(interp, sv) continue else @@ -206,7 +208,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), if (isa(sv, InferenceState) && infer_compilation_signature(interp) && (seenall && 1 == napplicable) && rettype !== Any && rettype !== Bottom && !is_removable_if_unused(all_effects)) - match = applicable[1]::MethodMatch + (; match) = applicable[1] method = match.method sig = match.spec_types mi = specialize_method(match; preexisting=true) @@ -243,7 +245,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), gfresult[] = CallMeta(rettype, exctype, all_effects, info, slotrefinements) return true - end # infercalls + end # function infercalls # start making progress on the first call infercalls(interp, sv) || push!(sv.tasks, infercalls) return gfresult @@ -253,8 +255,14 @@ struct FailedMethodMatch reason::String end +struct MethodMatchTarget + match::MethodMatch + edges::Vector{Union{Nothing,CodeInstance}} + edge_idx::Int +end + struct MethodMatches - applicable::Vector{Any} + applicable::Vector{MethodMatchTarget} info::MethodMatchInfo valid_worlds::WorldRange end @@ -265,7 +273,7 @@ fully_covering(info::MethodMatchInfo) = info.fullmatch fully_covering(m::MethodMatches) = fully_covering(m.info) struct UnionSplitMethodMatches - applicable::Vector{Any} + applicable::Vector{MethodMatchTarget} applicable_argtypes::Vector{Vector{Any}} info::UnionSplitInfo valid_worlds::WorldRange @@ -301,7 +309,7 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: @nospecialize(atype), max_methods::Int) split_argtypes = switchtupleunion(typeinf_lattice(interp), argtypes) infos = MethodMatchInfo[] - applicable = Any[] + applicable = MethodMatchTarget[] applicable_argtypes = Vector{Any}[] # arrays like `argtypes`, including constants, for each match valid_worlds = WorldRange() for i in 1:length(split_argtypes) @@ -314,14 +322,14 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: if thismatches === nothing return FailedMethodMatch("For one of the union split cases, too many methods matched") end - for m in thismatches - push!(applicable, m) - push!(applicable_argtypes, arg_n) - end valid_worlds = intersect(valid_worlds, thismatches.valid_worlds) thisfullmatch = any(match::MethodMatch->match.fully_covers, thismatches) thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch) push!(infos, thisinfo) + for idx = 1:length(thismatches) + push!(applicable, MethodMatchTarget(thismatches[idx], thisinfo.edges, idx)) + push!(applicable_argtypes, arg_n) + end end info = UnionSplitInfo(infos) return UnionSplitMethodMatches( @@ -342,7 +350,8 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a end fullmatch = any(match::MethodMatch->match.fully_covers, matches) info = MethodMatchInfo(matches, mt, atype, fullmatch) - return MethodMatches(matches.matches, info, matches.valid_worlds) + applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)] + return MethodMatches(applicable, info, matches.valid_worlds) end """ @@ -513,7 +522,7 @@ function conditional_argtype(𝕃ᡒ::AbstractLattice, @nospecialize(rt), @nospe end end -function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{Any}, +function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{MethodMatchTarget}, argtypes::Vector{Any}, fargs::Vector{Any}, sv::InferenceState) ⊏, βŠ” = strictpartialorder(𝕃ᡒ), join(𝕃ᡒ) slotrefinements = nothing @@ -527,7 +536,7 @@ function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{A end sigt = Bottom for j = 1:length(applicable) - match = applicable[j]::MethodMatch + (;match) = applicable[j] valid_as_lattice(match.spec_types, true) || continue sigt = sigt βŠ” fieldtype(match.spec_types, i) end @@ -551,9 +560,9 @@ function abstract_call_method(interp::AbstractInterpreter, hardlimit::Bool, si::StmtInfo, sv::AbsIntState) sigtuple = unwrap_unionall(sig) sigtuple isa DataType || - return Future(MethodCallResult(Any, Any, false, false, nothing, Effects())) + return Future(MethodCallResult(Any, Any, Effects(), nothing, false, false)) all(@nospecialize(x) -> valid_as_lattice(unwrapva(x), true), sigtuple.parameters) || - return Future(MethodCallResult(Union{}, Any, false, false, nothing, EFFECTS_THROWS)) # catch bad type intersections early + return Future(MethodCallResult(Union{}, Any, EFFECTS_THROWS, nothing, false, false)) # catch bad type intersections early if is_nospecializeinfer(method) sig = get_nospecializeinfer_sig(method, sig, sparams) @@ -578,7 +587,7 @@ function abstract_call_method(interp::AbstractInterpreter, # we have a self-cycle in the call-graph, but not in the inference graph (typically): # break this edge now (before we record it) by returning early # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases) - return Future(MethodCallResult(Any, Any, true, true, nothing, Effects())) + return Future(MethodCallResult(Any, Any, Effects(), nothing, true, true)) end topmost = nothing edgecycle = true @@ -633,7 +642,7 @@ function abstract_call_method(interp::AbstractInterpreter, # since it's very unlikely that we'll try to inline this, # or want make an invoke edge to its calling convention return type. # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases) - return Future(MethodCallResult(Any, Any, true, true, nothing, Effects())) + return Future(MethodCallResult(Any, Any, Effects(), nothing, true, true)) end add_remark!(interp, sv, washardlimit ? RECURSION_MSG_HARDLIMIT : RECURSION_MSG) # TODO (#48913) implement a proper recursion handling for irinterp: @@ -759,9 +768,9 @@ function matches_sv(parent::AbsIntState, sv::AbsIntState) method_for_inference_limit_heuristics(sv) === method_for_inference_limit_heuristics(parent)) end -function is_edge_recursed(edge::MethodInstance, caller::AbsIntState) +function is_edge_recursed(edge::CodeInstance, caller::AbsIntState) return any(AbsIntStackUnwind(caller)) do sv::AbsIntState - return edge === frame_instance(sv) + return edge.def === frame_instance(sv) end end @@ -788,18 +797,15 @@ end struct MethodCallResult rt exct + effects::Effects + edge::Union{Nothing,CodeInstance} edgecycle::Bool edgelimited::Bool - edge::Union{Nothing,MethodInstance} - effects::Effects volatile_inf_result::Union{Nothing,VolatileInferenceResult} - function MethodCallResult(@nospecialize(rt), @nospecialize(exct), - edgecycle::Bool, - edgelimited::Bool, - edge::Union{Nothing,MethodInstance}, - effects::Effects, + function MethodCallResult(@nospecialize(rt), @nospecialize(exct), effects::Effects, + edge::Union{Nothing,CodeInstance}, edgecycle::Bool, edgelimited::Bool, volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) - return new(rt, exct, edgecycle, edgelimited, edge, effects, volatile_inf_result) + return new(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result) end end @@ -809,12 +815,12 @@ struct InvokeCall InvokeCall(@nospecialize(types), @nospecialize(lookupsig)) = new(types, lookupsig) end -struct ConstCallResults +struct ConstCallResult rt::Any exct::Any const_result::ConstResult effects::Effects - function ConstCallResults( + function ConstCallResult( @nospecialize(rt), @nospecialize(exct), const_result::ConstResult, effects::Effects) @@ -901,8 +907,7 @@ function concrete_eval_eligible(interp::AbstractInterpreter, return :none end end - mi = result.edge - if mi !== nothing && is_foldable(effects, #=check_rtcall=#true) + if result.edge !== nothing && is_foldable(effects, #=check_rtcall=#true) if f !== nothing && is_all_const_arg(arginfo, #=start=#2) if (is_nonoverlayed(interp) || is_nonoverlayed(effects) || # Even if overlay methods are involved, when `:consistent_overlay` is @@ -964,15 +969,15 @@ function concrete_eval_call(interp::AbstractInterpreter, f = invoke end world = get_inference_world(interp) - edge = result.edge::MethodInstance + edge = result.edge::CodeInstance value = try Core._call_in_world_total(world, f, args...) catch e # The evaluation threw. By :consistent-cy, we're guaranteed this would have happened at runtime. # Howevever, at present, :consistency does not mandate the type of the exception - return ConstCallResults(Bottom, Any, ConcreteResult(edge, result.effects), result.effects) + return ConstCallResult(Bottom, Any, ConcreteResult(edge, result.effects), result.effects) end - return ConstCallResults(Const(value), Union{}, ConcreteResult(edge, EFFECTS_TOTAL, value), EFFECTS_TOTAL) + return ConstCallResult(Const(value), Union{}, ConcreteResult(edge, EFFECTS_TOTAL, value), EFFECTS_TOTAL) end # check if there is a cycle and duplicated inference of `mi` @@ -1216,9 +1221,9 @@ function semi_concrete_eval_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState) world = frame_world(sv) mi_cache = WorldView(code_cache(interp), world) - code = get(mi_cache, mi, nothing) - if code !== nothing - irsv = IRInterpretationState(interp, code, mi, arginfo.argtypes, world) + codeinst = get(mi_cache, mi, nothing) + if codeinst !== nothing + irsv = IRInterpretationState(interp, codeinst, mi, arginfo.argtypes, world) if irsv !== nothing assign_parentchild!(irsv, sv) rt, (nothrow, noub) = ir_abstract_constant_propagation(interp, irsv) @@ -1237,7 +1242,7 @@ function semi_concrete_eval_call(interp::AbstractInterpreter, effects = Effects(effects; noub=ALWAYS_TRUE) end exct = refine_exception_type(result.exct, effects) - return ConstCallResults(rt, exct, SemiConcreteResult(mi, ir, effects, spec_info(irsv)), effects) + return ConstCallResult(rt, exct, SemiConcreteResult(codeinst, ir, effects, spec_info(irsv)), effects) end end end @@ -1245,8 +1250,8 @@ function semi_concrete_eval_call(interp::AbstractInterpreter, end const_prop_result(inf_result::InferenceResult) = - ConstCallResults(inf_result.result, inf_result.exc_result, ConstPropResult(inf_result), - inf_result.ipo_effects) + ConstCallResult(inf_result.result, inf_result.exc_result, ConstPropResult(inf_result), + inf_result.ipo_effects) # return cached result of constant analysis return_localcache_result(::AbstractInterpreter, inf_result::InferenceResult, ::AbsIntState) = @@ -1259,7 +1264,7 @@ end function const_prop_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState, - concrete_eval_result::Union{Nothing, ConstCallResults}=nothing) + concrete_eval_result::Union{Nothing, ConstCallResult}=nothing) inf_cache = get_inference_cache(interp) 𝕃ᡒ = typeinf_lattice(interp) forwarded_argtypes = compute_forwarded_argtypes(interp, arginfo, sv) @@ -1645,7 +1650,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n end iterateresult[] = AbstractIterationResult(ret, AbstractIterationInfo(calls, false)) return true - end # inferiterate_2arg + end # function inferiterate_2arg # continue making progress as much as possible, on iterate(arg, state) inferiterate_2arg(interp, sv) || push!(sv.tasks, inferiterate_2arg) return true @@ -1815,7 +1820,7 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si:: # For now, only propagate info if we don't also union-split the iteration applyresult[] = CallMeta(res, exctype, all_effects, retinfo) return true - end + end # function infercalls # start making progress on the first call infercalls(interp, sv) || push!(sv.tasks, infercalls) return applyresult @@ -2184,7 +2189,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt mresult = abstract_call_method(interp, method, ti, env, false, si, sv)::Future match = MethodMatch(ti, env, method, argtype <: method.sig) return Future{CallMeta}(mresult, interp, sv) do result, interp, sv - (; rt, exct, effects, volatile_inf_result) = result + (; rt, exct, effects, edge, volatile_inf_result) = result res = nothing sig = match.spec_types argtypesβ€² = invoke_rewrite(argtypes) @@ -2204,6 +2209,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt result, f, arginfo, si, match, sv, invokecall) const_result = volatile_inf_result if const_call_result !== nothing + # TODO override the edge with the const-prop' edge if const_call_result.rt βŠ‘ rt (; rt, effects, const_result) = const_call_result end @@ -2212,7 +2218,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt end end rt = from_interprocedural!(interp, rt, sv, arginfo, sig) - info = InvokeCallInfo(match, const_result, lookupsig) + info = InvokeCallInfo(edge, match, const_result, lookupsig) if !match.fully_covers effects = Effects(effects; nothrow=false) exct = exct βŠ” TypeError @@ -2281,7 +2287,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return abstract_apply(interp, argtypes, si, sv, max_methods) elseif f === invoke return abstract_invoke(interp, arginfo, si, sv) - elseif f === modifyfield! || f === Core.modifyglobal! || f === Core.memoryrefmodify! || f === atomic_pointermodify + elseif f === modifyfield! || f === Core.modifyglobal! || + f === Core.memoryrefmodify! || f === atomic_pointermodify return abstract_modifyop!(interp, f, argtypes, si, sv) elseif f === Core.finalizer return abstract_finalizer(interp, argtypes, sv) @@ -2408,7 +2415,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, mresult = abstract_call_method(interp, ocmethod, sig, Core.svec(), false, si, sv) ocsig_box = Core.Box(ocsig) return Future{CallMeta}(mresult, interp, sv) do result, interp, sv - (; rt, exct, effects, volatile_inf_result, edgecycle) = result + (; rt, exct, effects, volatile_inf_result, edge, edgecycle) = result π•ƒβ‚š = ipo_lattice(interp) βŠ‘, β‹€, βŠ” = partialorder(π•ƒβ‚š), strictneqpartialorder(π•ƒβ‚š), join(π•ƒβ‚š) const_result = volatile_inf_result @@ -2416,6 +2423,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, const_call_result = abstract_call_method_with_const_args(interp, result, #=f=#nothing, arginfo, si, match, sv) if const_call_result !== nothing + # TODO override the edge with the const-prop' edge if const_call_result.rt βŠ‘ rt (; rt, effects, const_result) = const_call_result end @@ -2434,7 +2442,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, end end rt = from_interprocedural!(interp, rt, sv, arginfo, match.spec_types) - info = OpaqueClosureCallInfo(match, const_result) + info = OpaqueClosureCallInfo(edge, match, const_result) return CallMeta(rt, exct, effects, info) end end diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 2f1d7dc279d70..cd6cd1ae5626d 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -68,7 +68,8 @@ struct InliningEdgeTracker new(state.edges, invokesig) end -function add_inlining_edge!((; edges, invokesig)::InliningEdgeTracker, mi::MethodInstance) +function add_inlining_edge!(et::InliningEdgeTracker, mi::MethodInstance) + (; edges, invokesig) = et if invokesig === nothing add_one_edge!(edges, mi) else # invoke backedge @@ -1119,7 +1120,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}}, # e.g. rewrite `((t::Tuple)...,)` to `t` nonempty_idx = 0 𝕃ₒ = optimizer_lattice(state.interp) - for i = (arg_start + 1):length(argtypes) + for i = (arg_start+1):length(argtypes) ti = argtypes[i] βŠ‘(𝕃ₒ, ti, Tuple{}) && continue if βŠ‘(𝕃ₒ, ti, Tuple) && nonempty_idx == 0 @@ -1137,7 +1138,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}}, # Try to figure out the signature of the function being called # and if rewrite_apply_exprargs can deal with this form arginfos = MaybeAbstractIterationInfo[] - for i = (arg_start + 1):length(argtypes) + for i = (arg_start+1):length(argtypes) thisarginfo = nothing if !is_valid_type_for_apply_rewrite(argtypes[i], OptimizationParams(state.interp)) isa(info, ApplyCallInfo) || return nothing @@ -1455,7 +1456,7 @@ end function semiconcrete_result_item(result::SemiConcreteResult, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) - mi = result.mi + mi = result.edge.def et = InliningEdgeTracker(state) if (!OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) || @@ -1478,7 +1479,7 @@ end function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult, match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) - mi = result.mi + mi = result.edge.def spec_types = match.spec_types validate_sparams(mi.sparam_vals) || return false item = semiconcrete_result_item(result, info, flag, state) @@ -1502,7 +1503,7 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn invokesig::Union{Nothing,Vector{Any}}=nothing) if !may_inline_concrete_result(result) et = InliningEdgeTracker(state, invokesig) - return compileable_specialization(result.mi, result.effects, et, info; + return compileable_specialization(result.edge.def, result.effects, et, info; compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) end @assert result.effects === EFFECTS_TOTAL diff --git a/base/compiler/stmtinfo.jl b/base/compiler/stmtinfo.jl index 95e6dc156b374..25f6a10cf47da 100644 --- a/base/compiler/stmtinfo.jl +++ b/base/compiler/stmtinfo.jl @@ -37,51 +37,58 @@ struct MethodMatchInfo <: CallInfo mt::MethodTable atype fullmatch::Bool + edges::Vector{Union{Nothing,CodeInstance}} + function MethodMatchInfo( + results::MethodLookupResult, mt::MethodTable, @nospecialize(atype), fullmatch::Bool) + edges = fill!(Vector{Union{Nothing,CodeInstance}}(undef, length(results)), nothing) + return new(results, mt, atype, fullmatch, edges) + end end function add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) - matches = info.results.matches if !fully_covering(info) # add legacy-style missing backedge info also exists = false for i in 1:length(edges) - if edges[i] === info.mt && edges[i + 1] == info.atype + if edges[i] === info.mt && edges[i+1] == info.atype exists = true break end end if !exists - push!(edges, info.mt) - push!(edges, info.atype) + push!(edges, info.mt, info.atype) end end - if length(matches) == 1 - # try the optimized format for the representation, if possible and applicable - # if this doesn't succeed, the backedge will be less precise, - # but the forward edge will maintain the precision - m = matches[1]::Core.MethodMatch - mi = specialize_method(m) - if mi.specTypes === m.spec_types - add_one_edge!(edges, mi) - return nothing + nmatches = length(info.results) + if nmatches == length(info.edges) == 1 + edge = info.edges[1] + if edge !== nothing + # try the optimized format for the representation, if possible and applicable + # if this doesn't succeed, the backedge will be less precise, + # but the forward edge will maintain the precision + if edge.def.specTypes === info.results[1].spec_types + add_one_edge!(edges, edge.def) # TODO CodeInstance as edge + return nothing + end end end # add check for whether this lookup already existed in the edges list for i in 1:length(edges) - if edges[i] === length(matches) && edges[i + 1] == info.atype + if edges[i] === nmatches && edges[i+1] == info.atype return nothing end end - push!(edges, length(matches)) - push!(edges, info.atype) - for m in matches - mi = specialize_method(m::Core.MethodMatch) - push!(edges, mi) + push!(edges, nmatches, info.atype) + for i = 1:nmatches + edge = info.edges[i] + if edge !== nothing + push!(edges, edge.def) # TODO CodeInstance as edge + end end nothing end function add_one_edge!(edges::Vector{Any}, mi::MethodInstance) for i in 1:length(edges) - if edges[i] === mi && !(i > 1 && edges[i - 1] isa Type) + if edges[i] === mi && !(i > 1 && edges[i-1] isa Type) return end end @@ -137,15 +144,15 @@ struct ConstPropResult <: ConstResult end struct ConcreteResult <: ConstResult - mi::MethodInstance + edge::CodeInstance effects::Effects result - ConcreteResult(mi::MethodInstance, effects::Effects) = new(mi, effects) - ConcreteResult(mi::MethodInstance, effects::Effects, @nospecialize val) = new(mi, effects, val) + ConcreteResult(edge::CodeInstance, effects::Effects) = new(edge, effects) + ConcreteResult(edge::CodeInstance, effects::Effects, @nospecialize val) = new(edge, effects, val) end struct SemiConcreteResult <: ConstResult - mi::MethodInstance + edge::CodeInstance ir::IRCode effects::Effects spec_info::SpecInfo @@ -228,8 +235,6 @@ function add_edges_impl(edges::Vector{Any}, info::ApplyCallInfo) end end end -# N.B. `ApplyCallInfo` doesn't need to implement the interfaces for the inlining, -# since `Core._apply_iterate` is handled by the special case """ info::UnionSplitApplyCallInfo <: CallInfo @@ -242,8 +247,6 @@ struct UnionSplitApplyCallInfo <: CallInfo end add_edges_impl(edges::Vector{Any}, info::UnionSplitApplyCallInfo) = for split in info.infos; add_edges!(edges, split); end -# N.B. `UnionSplitApplyCallInfo` doesn't need to implement the interfaces for the inlining, -# since `Core._apply_iterate` is handled by the special case """ info::InvokeCallInfo @@ -253,24 +256,29 @@ the method that has been processed. Optionally keeps `info.result::InferenceResult` that keeps constant information. """ struct InvokeCallInfo <: CallInfo + edge::Union{Nothing,CodeInstance} match::MethodMatch result::Union{Nothing,ConstResult} atype # ::Type end -add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo) = - add_invoke_edge!(edges, info.atype, specialize_method(info.match)) +function add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo) + edge = info.edge + if edge !== nothing + add_invoke_edge!(edges, info.atype, edge.def) # TODO CodeInstance as edge + end +end function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), mi::MethodInstance) for i in 2:length(edges) - if edges[i] === mi && edges[i - 1] isa Type && edges[i - 1] == atype - return + if edges[i] === mi + edge_minus_1 = edges[i-1] + if edge_minus_1 isa Type && edge_minus_1 == atype + return nothing + end end end - push!(edges, atype) - push!(edges, mi) + push!(edges, atype, mi) nothing end -# N.B. `InvokeCallInfo` doesn't need to implement the interfaces for the inlining, -# since `Core.invoke` is handled by the special case """ info::OpaqueClosureCallInfo @@ -280,13 +288,16 @@ the method that has been processed. Optionally keeps `info.result::InferenceResult` that keeps constant information. """ struct OpaqueClosureCallInfo <: CallInfo + edge::Union{Nothing,CodeInstance} match::MethodMatch result::Union{Nothing,ConstResult} end -add_edges_impl(edges::Vector{Any}, info::OpaqueClosureCallInfo) = - add_one_edge!(edges, specialize_method(info.match)) -# N.B. `OpaqueClosureCallInfo` doesn't need to implement the interfaces for the inlining, -# since `Core.invoke` is handled by the special case +function add_edges_impl(edges::Vector{Any}, info::OpaqueClosureCallInfo) + edge = info.edge + if edge !== nothing + add_one_edge!(edges, edge.def) # TODO CodeInstance as edge + end +end """ info::OpaqueClosureCreateInfo <: CallInfo @@ -352,4 +363,9 @@ struct ModifyOpInfo <: CallInfo end add_edges_impl(edges::Vector{Any}, info::ModifyOpInfo) = add_edges!(edges, info.info) +struct VirtualMethodMatchInfo <: CallInfo + info::CallInfo +end +add_edges_impl(edges::Vector{Any}, info::VirtualMethodMatchInfo) = add_edges!(edges, info.info) + @specialize diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index f4c7ca94dd669..cd8c0443e056f 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -3010,7 +3010,11 @@ function abstract_applicable(interp::AbstractInterpreter, argtypes::Vector{Any}, rt = Const(true) # has applicable matches end if rt !== Bool - info = MethodResultPure(matches.info) # XXX this should probably be something like + for i = 1:napplicable + (; match, edges, edge_idx) = applicable[i] + edges[edge_idx] = codeinstance_for_method_match_edge(interp, specialize_method(match)) + end + info = VirtualMethodMatchInfo(matches.info) end end return Future(CallMeta(rt, Union{}, EFFECTS_TOTAL, info)) @@ -3051,12 +3055,20 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge else rt = Const(true) - vinfo = InvokeCallInfo(match, nothing, types) + edge = codeinstance_for_method_match_edge(interp, specialize_method(match)) + vinfo = InvokeCallInfo(edge, match, nothing, types) end - info = MethodResultPure(vinfo) # XXX this should probably be something like `VirtualizedCallInfo` + info = VirtualMethodMatchInfo(vinfo) return CallMeta(rt, Union{}, EFFECTS_TOTAL, info) end +# allocate a dummy `edge::CodeInstance` to be added by `add_edges!` +function codeinstance_for_method_match_edge(interp::AbstractInterpreter, edge::MethodInstance) + return CodeInstance(edge, cache_owner(interp), Any, Any, nothing, nothing, zero(Int32), + get_inference_world(interp), get_inference_world(interp), + zero(UInt32), nothing, zero(UInt8), nothing, empty_edges) +end + # N.B.: typename maps type equivalence classes to a single value function typename_static(@nospecialize(t)) t isa Const && return _typename(t.val) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 80fcff9068da1..3ab52b357d94e 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -105,11 +105,12 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState; # but our caller might, so let's just make it anyways store_backedges(result, caller.edges) end - isa(caller.linfo.def, Method) || empty!(caller.edges) # don't add backedges to toplevel method instance if isdefined(result, :ci) ci = result.ci inferred_result = nothing - edges = Core.svec(caller.edges...) + # don't add backedges to toplevel method instance + istoplevel = caller.linfo.def isa Method + edges = istoplevel ? empty_edges : Core.svec(caller.edges...) relocatability = 0x1 const_flag = is_result_constabi_eligible(result) if !can_discard_trees || (is_cached(caller) && !const_flag) @@ -251,19 +252,15 @@ function cache_result!(interp::AbstractInterpreter, result::InferenceResult) # check if the existing linfo metadata is also sufficient to describe the current inference result # to decide if it is worth caching this right now mi = result.linfo - cache_results = true cache = WorldView(code_cache(interp), result.valid_worlds) - if cache_results && haskey(cache, mi) + if haskey(cache, mi) ci = cache[mi] # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later @assert isdefined(ci, :inferred) - cache_results = false + return false end - - if cache_results - code_cache(interp)[mi] = result.ci - end - return cache_results + code_cache(interp)[mi] = result.ci + return true end function cycle_fix_limited(@nospecialize(typ), sv::InferenceState) @@ -389,6 +386,8 @@ function refine_exception_type(@nospecialize(exc_bestguess), ipo_effects::Effect return exc_bestguess end +const empty_edges = Core.svec() + # inference completed on `me` # update the MethodInstance function finishinfer!(me::InferenceState, interp::AbstractInterpreter) @@ -418,9 +417,8 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) me.src.rettype = widenconst(ignorelimited(bestguess)) me.src.min_world = first(me.valid_worlds) me.src.max_world = last(me.valid_worlds) - if isa(me.linfo.def, Method) # don't add backedges to toplevel method instance - compute_edges!(me) - end + istoplevel = me.linfo.def isa Method + istoplevel && compute_edges!(me) # don't add backedges to toplevel method instance if limited_ret # a parent may be cached still, but not this intermediate work: @@ -451,7 +449,6 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) end maybe_validate_code(me.linfo, me.src, "inferred") - isa(me.linfo.def, Method) || empty!(me.edges) # don't add backedges to toplevel method instance # finish populating inference results into the CodeInstance if possible, and maybe cache that globally for use elsewhere if isdefined(result, :ci) && !limited_ret @@ -477,18 +474,18 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) const_flags = 0x2 else rettype_const = nothing - const_flags = 0x00 + const_flags = 0x0 end relocatability = 0x0 di = nothing - edges = Core.svec(me.edges...) + edges = istoplevel ? empty_edges : Core.svec(me.edges...) ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), result.ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), result.analysis_results, di, edges) if is_cached(me) - cached_results = cache_result!(me.interp, me.result) - if !cached_results + cached_result = cache_result!(me.interp, me.result) + if !cached_result me.cache_mode = CACHE_MODE_NULL end end @@ -505,7 +502,6 @@ function store_backedges(caller::MethodInstance, edges::Vector{Any}) if isa(callee, MethodInstance) ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, itr.sig, caller) else - typeassert(callee, MethodTable) ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), callee, itr.sig, caller) end end @@ -517,7 +513,7 @@ function compute_edges!(sv::InferenceState) for i in 1:length(sv.stmt_info) add_edges!(edges, sv.stmt_info[i]) end - if sv.src.edges !== nothing && sv.src.edges !== Core.svec() + if sv.src.edges !== nothing && sv.src.edges !== empty_edges append!(edges, sv.src.edges) end nothing @@ -697,31 +693,20 @@ end ipo_effects(code::CodeInstance) = decode_effects(code.ipo_purity_bits) -struct EdgeCallResult - rt - exct - edge::Union{Nothing,MethodInstance} - effects::Effects - volatile_inf_result::Union{Nothing,VolatileInferenceResult} - function EdgeCallResult(@nospecialize(rt), @nospecialize(exct), - edge::Union{Nothing,MethodInstance}, - effects::Effects, - volatile_inf_result::Union{Nothing,VolatileInferenceResult} = nothing) - return new(rt, exct, edge, effects, volatile_inf_result) - end -end - # return cached result of regular inference function return_cached_result(interp::AbstractInterpreter, method::Method, codeinst::CodeInstance, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool) rt = cached_return_type(codeinst) + exct = codeinst.exctype effects = ipo_effects(codeinst) + edge = codeinst update_valid_age!(caller, WorldRange(min_world(codeinst), max_world(codeinst))) - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(rt, codeinst.exctype, codeinst.def, effects), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, rt, exct, effects, edge, edgecycle, edgelimited)) end -function EdgeCall_to_MethodCall_Result(interp::AbstractInterpreter, sv::AbsIntState, method::Method, result::EdgeCallResult, edgecycle::Bool, edgelimited::Bool) - (; rt, exct, edge, effects, volatile_inf_result) = result - +function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method, + @nospecialize(rt), @nospecialize(exct), effects::Effects, + edge::Union{Nothing,CodeInstance}, edgecycle::Bool, edgelimited::Bool, + volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) if edge === nothing edgecycle = edgelimited = true end @@ -744,12 +729,12 @@ function EdgeCall_to_MethodCall_Result(interp::AbstractInterpreter, sv::AbsIntSt end end - return MethodCallResult(rt, exct, edgecycle, edgelimited, edge, effects, volatile_inf_result) + return MethodCallResult(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result) end # compute (and cache) an inferred AST and return the current best estimate of the result type function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool) - mi = specialize_method(method, atype, sparams)::MethodInstance + mi = specialize_method(method, atype, sparams) cache_mode = CACHE_MODE_GLOBAL # cache edge targets globally by default force_inline = is_stmt_inline(get_curr_ssaflag(caller)) let codeinst = get(code_cache(interp), mi, nothing) @@ -768,7 +753,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize end if ccall(:jl_get_module_infer, Cint, (Any,), method.module) == 0 && !generating_output(#=incremental=#false) add_remark!(interp, caller, "[typeinf_edge] Inference is disabled for the target module") - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(Any, Any, nothing, Effects()), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end if !is_cached(caller) && frame_parent(caller) === nothing # this caller exists to return to the user @@ -779,33 +764,33 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize end if frame === false # completely new, but check again after reserving in the engine - if cache_mode == CACHE_MODE_GLOBAL - ci = engine_reserve(interp, mi) - let codeinst = get(code_cache(interp), mi, nothing) - if codeinst isa CodeInstance # return existing rettype if the code is already inferred - engine_reject(interp, ci) - inferred = @atomic :monotonic codeinst.inferred - if inferred === nothing && force_inline - cache_mode = CACHE_MODE_VOLATILE - else - @assert codeinst.def === mi "MethodInstance for cached edge does not match" - return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) - end + ci_from_engine = cache_mode == CACHE_MODE_GLOBAL ? engine_reserve(interp, mi) : nothing + if ci_from_engine !== nothing + codeinst = get(code_cache(interp), mi, nothing) + if codeinst isa CodeInstance # return existing rettype if the code is already inferred + engine_reject(interp, ci_from_engine) + ci_from_engine = nothing + inferred = @atomic :monotonic codeinst.inferred + if inferred === nothing && force_inline + cache_mode = CACHE_MODE_VOLATILE + else + @assert codeinst.def === mi "MethodInstance for cached edge does not match" + return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) end end end result = InferenceResult(mi, typeinf_lattice(interp)) - if cache_mode == CACHE_MODE_GLOBAL - result.ci = ci + if ci_from_engine !== nothing + result.ci = ci_from_engine end frame = InferenceState(result, cache_mode, interp) # always use the cache for edge targets if frame === nothing add_remark!(interp, caller, "[typeinf_edge] Failed to retrieve source") # can't get the source for this, so we know nothing - if cache_mode == CACHE_MODE_GLOBAL - engine_reject(interp, ci) + if ci_from_engine !== nothing + engine_reject(interp, ci_from_engine) end - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(Any, Any, nothing, Effects()), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end assign_parentchild!(frame, caller) # the actual inference task for this edge is going to be scheduled within `typeinf_local` via the callstack queue @@ -814,15 +799,17 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize push!(caller.tasks, function get_infer_result(interp, caller) update_valid_age!(caller, frame.valid_worlds) local isinferred = is_inferred(frame) - local edge = isinferred ? mi : nothing + # TODO allocate an edge when isinferred && ci_from_engine === nothing (i.e. cache_mode === CACHE_MODE_LOCAL) + local edge = isinferred ? ci_from_engine : nothing local effects = isinferred ? frame.result.ipo_effects : # effects are adjusted already within `finish` for ipo_effects adjust_effects(effects_for_cycle(frame.ipo_effects), method) + local bestguess = frame.bestguess local exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) # propagate newly inferred source to the inliner, allowing efficient inlining w/o deserialization: # note that this result is cached globally exclusively, so we can use this local result destructively local volatile_inf_result = isinferred ? VolatileInferenceResult(result) : nothing - local edgeresult = EdgeCallResult(frame.bestguess, exc_bestguess, edge, effects, volatile_inf_result) - mresult[] = EdgeCall_to_MethodCall_Result(interp, caller, method, edgeresult, edgecycle, edgelimited) + mresult[] = MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, + edge, edgecycle, edgelimited, volatile_inf_result) return true end) return mresult @@ -830,15 +817,15 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize elseif frame === true # unresolvable cycle add_remark!(interp, caller, "[typeinf_edge] Unresolvable cycle") - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(Any, Any, nothing, Effects()), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end # return the current knowledge about this cycle frame = frame::InferenceState update_valid_age!(caller, frame.valid_worlds) effects = adjust_effects(effects_for_cycle(frame.ipo_effects), method) + bestguess = frame.bestguess exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) - edgeresult = EdgeCallResult(frame.bestguess, exc_bestguess, nothing, effects) - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, edgeresult, edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, nothing, edgecycle, edgelimited)) end # The `:terminates` effect bit must be conservatively tainted unless recursion cycle has diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index 21af923f8f30c..3c286aa742891 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -252,8 +252,6 @@ struct BackedgeIterator backedges::Vector{Any} end -const empty_backedge_iter = BackedgeIterator(Any[]) - struct BackedgePair sig # ::Union{Nothing,Type} caller::Union{MethodInstance,MethodTable} diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index d59a18e6d4f16..69e6137010902 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -673,8 +673,8 @@ function CC.concrete_eval_eligible(interp::REPLInterpreter, @nospecialize(f), sv::CC.InferenceState) if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) neweffects = CC.Effects(result.effects; consistent=CC.ALWAYS_TRUE) - result = CC.MethodCallResult(result.rt, result.exct, result.edgecycle, result.edgelimited, - result.edge, neweffects) + result = CC.MethodCallResult(result.rt, result.exct, neweffects, result.edge, + result.edgecycle, result.edgelimited, result.volatile_inf_result) end ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo,