diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index cd6cd1ae5626d..a95ad958e8e00 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -28,7 +28,8 @@ end struct ConstantCase val::Any - ConstantCase(@nospecialize val) = new(val) + edge::CodeInstance + ConstantCase(@nospecialize(val), edge::CodeInstance) = new(val, edge) end struct SomeCase @@ -68,12 +69,12 @@ struct InliningEdgeTracker new(state.edges, invokesig) end -function add_inlining_edge!(et::InliningEdgeTracker, mi::MethodInstance) +function add_inlining_edge!(et::InliningEdgeTracker, edge::CodeInstance) (; edges, invokesig) = et if invokesig === nothing - add_one_edge!(edges, mi) + add_one_edge!(edges, edge) else # invoke backedge - add_invoke_edge!(edges, invoke_signature(invokesig), mi) + add_invoke_edge!(edges, invoke_signature(invokesig), edge) end return nothing end @@ -784,11 +785,11 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}}, return new_argtypes end -function compileable_specialization(mi::MethodInstance, effects::Effects, - et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true) - mi_invoke = mi +function compileable_specialization(edge::CodeInstance, effects::Effects, + et::InliningEdgeTracker, @nospecialize(info::CallInfo), state::InliningState) + mi_invoke = mi = edge.def method, atype, sparams = mi.def::Method, mi.specTypes, mi.sparam_vals - if compilesig_invokes + if OptimizationParams(state.interp).compilesig_invokes new_atype = get_compileable_sig(method, atype, sparams) new_atype === nothing && return nothing if atype !== new_atype @@ -806,43 +807,42 @@ function compileable_specialization(mi::MethodInstance, effects::Effects, return nothing end end - add_inlining_edge!(et, mi) # to the dispatch lookup - mi_invoke !== mi && add_invoke_edge!(et.edges, method.sig, mi_invoke) # add_inlining_edge to the invoke call, if that is different + add_inlining_edge!(et, edge) # to the dispatch lookup + if mi_invoke !== mi + edge_invoke = codeinst_as_invoke_edge(state.interp, mi_invoke) + add_invoke_edge!(et.edges, method.sig, edge_invoke) # add_inlining_edge to the invoke call, if that is different + end return InvokeCase(mi_invoke, effects, info) end -function compileable_specialization(match::MethodMatch, effects::Effects, - et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true) - mi = specialize_method(match) - return compileable_specialization(mi, effects, et, info; compilesig_invokes) -end - struct InferredResult src::Any # CodeInfo or IRCode effects::Effects - InferredResult(@nospecialize(src), effects::Effects) = new(src, effects) + edge::CodeInstance + InferredResult(@nospecialize(src), effects::Effects, edge::CodeInstance) = new(src, effects, edge) end @inline function get_cached_result(state::InliningState, mi::MethodInstance) code = get(code_cache(state), mi, nothing) if code isa CodeInstance if use_const_api(code) # in this case function can be inlined to a constant - return ConstantCase(quoted(code.rettype_const)) + return ConstantCase(quoted(code.rettype_const), code) end return code end return nothing end @inline function get_local_result(inf_result::InferenceResult) + @assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge" effects = inf_result.ipo_effects if is_foldable_nothrow(effects) res = inf_result.result if isa(res, Const) && is_inlineable_constant(res.val) # use constant calling convention - return ConstantCase(quoted(res.val)) + return ConstantCase(quoted(res.val), inf_result.ci_as_edge) end end - return InferredResult(inf_result.src, effects) + return InferredResult(inf_result.src, effects, inf_result.ci_as_edge) end # the general resolver for usual and const-prop'ed calls @@ -862,30 +862,29 @@ function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult, inferred_result = get_cached_result(state, mi) end if inferred_result isa ConstantCase - add_inlining_edge!(et, mi) + add_inlining_edge!(et, inferred_result.edge) return inferred_result elseif inferred_result isa InferredResult - (; src, effects) = inferred_result + (; src, effects, edge) = inferred_result elseif inferred_result isa CodeInstance src = @atomic :monotonic inferred_result.inferred effects = decode_effects(inferred_result.ipo_purity_bits) + edge = inferred_result else # there is no cached source available, bail out - return compileable_specialization(mi, Effects(), et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + edge = codeinst_as_invoke_edge(state.interp, mi) + return compileable_specialization(edge, Effects(), et, info, state) end # the duplicated check might have been done already within `analyze_method!`, but still # we need it here too since we may come here directly using a constant-prop' result if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) - return compileable_specialization(mi, effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(edge, effects, et, info, state) end src_inlining_policy(state.interp, src, info, flag) || - return compileable_specialization(mi, effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(edge, effects, et, info, state) - add_inlining_edge!(et, mi) + add_inlining_edge!(et, edge) if inferred_result isa CodeInstance ir, spec_info, debuginfo = retrieve_ir_for_inlining(inferred_result, src) else @@ -905,7 +904,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U cached_result = get_cached_result(state, mi) if cached_result isa ConstantCase - add_inlining_edge!(et, mi) + add_inlining_edge!(et, cached_result.edge) return cached_result elseif cached_result isa CodeInstance src = @atomic :monotonic cached_result.inferred @@ -916,7 +915,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U src_inlining_policy(state.interp, src, info, flag) || return nothing ir, spec_info, debuginfo = retrieve_ir_for_inlining(cached_result, src) - add_inlining_edge!(et, mi) + add_inlining_edge!(et, cached_result) return InliningTodo(mi, ir, spec_info, debuginfo, effects) end @@ -1464,14 +1463,12 @@ function semiconcrete_result_item(result::SemiConcreteResult, # a `@noinline`-declared method when it's marked as `@constprop :aggressive`. # Suppress the inlining here (unless inlining is requested at the callsite). (is_declared_noinline(mi.def::Method) && !is_stmt_inline(flag))) - return compileable_specialization(mi, result.effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(result.edge, result.effects, et, info, state) end src_inlining_policy(state.interp, result.ir, info, flag) || - return compileable_specialization(mi, result.effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(result.edge, result.effects, et, info, state) - add_inlining_edge!(et, mi) + add_inlining_edge!(et, result.edge) preserve_local_sources = OptimizationParams(state.interp).preserve_local_sources ir, _, debuginfo = retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) return InliningTodo(mi, ir, result.spec_info, debuginfo, result.effects) @@ -1503,11 +1500,10 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn invokesig::Union{Nothing,Vector{Any}}=nothing) if !may_inline_concrete_result(result) et = InliningEdgeTracker(state, invokesig) - return compileable_specialization(result.edge.def, result.effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(result.edge, result.effects, et, info, state) end @assert result.effects === EFFECTS_TOTAL - return ConstantCase(quoted(result.result)) + return ConstantCase(quoted(result.result), result.edge) end function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @@ -1556,11 +1552,14 @@ function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOp info isa MethodResultPure && (info = info.info) info isa ConstCallInfo && (info = info.call) info isa MethodMatchInfo || return nothing - length(info.results) == 1 || return nothing + length(info.edges) == length(info.results) == 1 || return nothing match = info.results[1]::MethodMatch match.fully_covers || return nothing - case = compileable_specialization(match, Effects(), InliningEdgeTracker(state), info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + edge = info.edges[1] + if edge === nothing + edge = codeinst_as_invoke_edge(state.interp, specialize_method(match)) + end + case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state) case === nothing && return nothing stmt.head = :invoke_modify pushfirst!(stmt.args, case.invoke) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index d7531f0f411a6..a81949a398410 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1529,7 +1529,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, if code isa CodeInstance if use_const_api(code) # No code in the function - Nothing to do - add_inlining_edge!(et, mi) + add_inlining_edge!(et, code) return true end src = @atomic :monotonic code.inferred @@ -1544,7 +1544,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, length(src.cfg.blocks) == 1 || return false # Ok, we're committed to inlining the finalizer - add_inlining_edge!(et, mi) + add_inlining_edge!(et, code) # TODO: Should there be a special line number node for inlined finalizers? inline_at = ir[SSAValue(idx)][:line] diff --git a/base/compiler/stmtinfo.jl b/base/compiler/stmtinfo.jl index 25f6a10cf47da..32209b805668f 100644 --- a/base/compiler/stmtinfo.jl +++ b/base/compiler/stmtinfo.jl @@ -66,7 +66,7 @@ function add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) # if this doesn't succeed, the backedge will be less precise, # but the forward edge will maintain the precision if edge.def.specTypes === info.results[1].spec_types - add_one_edge!(edges, edge.def) # TODO CodeInstance as edge + add_one_edge!(edges, edge) return nothing end end @@ -81,18 +81,20 @@ function add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) for i = 1:nmatches edge = info.edges[i] if edge !== nothing - push!(edges, edge.def) # TODO CodeInstance as edge + push!(edges, edge) end end nothing end -function add_one_edge!(edges::Vector{Any}, mi::MethodInstance) +function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) for i in 1:length(edges) - if edges[i] === mi && !(i > 1 && edges[i-1] isa Type) + edgeᵢ = edges[i] + # XXX compare `CodeInstance` identify? + if edgeᵢ isa CodeInstance && edgeᵢ.def === edge.def && !(i > 1 && edges[i-1] isa Type) return end end - push!(edges, mi) + push!(edges, edge) nothing end nsplit_impl(info::MethodMatchInfo) = 1 @@ -264,19 +266,21 @@ end function add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo) edge = info.edge if edge !== nothing - add_invoke_edge!(edges, info.atype, edge.def) # TODO CodeInstance as edge + add_invoke_edge!(edges, info.atype, edge) end + nothing end -function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), mi::MethodInstance) +function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::CodeInstance) for i in 2:length(edges) - if edges[i] === mi + edgeᵢ = edges[i] + if edgeᵢ isa CodeInstance && edgeᵢ.def === edge.def # XXX compare `CodeInstance` identify? edge_minus_1 = edges[i-1] if edge_minus_1 isa Type && edge_minus_1 == atype return nothing end end end - push!(edges, atype, mi) + push!(edges, atype, edge) nothing end @@ -295,7 +299,7 @@ end function add_edges_impl(edges::Vector{Any}, info::OpaqueClosureCallInfo) edge = info.edge if edge !== nothing - add_one_edge!(edges, edge.def) # TODO CodeInstance as edge + add_one_edge!(edges, edge) end end diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index eecd8e1287ab6..8b852617a312c 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1432,7 +1432,7 @@ end return Future{CallMeta}(callinfo, interp, sv) do callinfo, interp, sv TF = TF.contents RT = RT.contents - TF2 = tmeet(callinfo.rt, widenconst(TF)) + TF2 = tmeet(ipo_lattice(interp), callinfo.rt, widenconst(TF)) if TF2 === Bottom RT = Bottom elseif isconcretetype(RT) && has_nontrivial_extended_info(𝕃ᵢ, TF2) # isconcrete condition required to form a PartialStruct @@ -3012,7 +3012,7 @@ function abstract_applicable(interp::AbstractInterpreter, argtypes::Vector{Any}, if rt !== Bool for i = 1:napplicable (; match, edges, edge_idx) = applicable[i] - edges[edge_idx] = codeinst_as_method_match_edge(interp, specialize_method(match)) + edges[edge_idx] = codeinst_as_invoke_edge(interp, specialize_method(match)) end info = VirtualMethodMatchInfo(matches.info) end @@ -3055,7 +3055,7 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge else rt = Const(true) - edge = codeinst_as_method_match_edge(interp, specialize_method(match)) + edge = codeinst_as_invoke_edge(interp, specialize_method(match)) vinfo = InvokeCallInfo(edge, match, nothing, types) end info = VirtualMethodMatchInfo(vinfo) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index d980466cf56ed..456aacf873022 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -134,7 +134,13 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState; di = DebugInfo(result.linfo) end istoplevel = caller.linfo.def isa Method # don't add backedges to toplevel method instance - edges = istoplevel ? empty_edges : Core.svec(caller.edges...) + if istoplevel + edges = empty_edges + else + # TODO remove the next line once we make staticdata_utils.c able to handle `CodeInstance` edge directly + edges = Any[edge isa CodeInstance ? edge.def : edge for edge in caller.edges] + edges = Core.svec(edges...) + end min_world, max_world = first(valid_worlds), last(valid_worlds) effects_bits = encode_effects(result.ipo_effects) ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any), @@ -736,12 +742,15 @@ function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method end # allocate a dummy `edge::CodeInstance` to be added by `add_edges!` -function codeinst_as_edge(interp::AbstractInterpreter, edge::MethodInstance, edges::SimpleVector) - return CodeInstance(edge, cache_owner(interp), Any, Any, nothing, nothing, zero(Int32), - get_inference_world(interp), get_inference_world(interp), - zero(UInt32), nothing, zero(UInt8), nothing, edges) +function codeinst_as_edge(edge::MethodInstance, edges::SimpleVector, @nospecialize(owner), min_world::UInt, max_world::UInt) + return CodeInstance(edge, owner, Any, Any, nothing, nothing, zero(Int32), + min_world, max_world, zero(UInt32), nothing, zero(UInt8), nothing, edges) end -codeinst_as_method_match_edge(interp::AbstractInterpreter, edge::MethodInstance) = +codeinst_as_edge(interp::AbstractInterpreter, edge::MethodInstance, edges::SimpleVector) = + codeinst_as_edge(edge, edges, cache_owner(interp), get_inference_world(interp), get_inference_world(interp)) +codeinst_as_invoke_edge(edge::MethodInstance, @nospecialize(owner), min_world::UInt, max_world::UInt) = + codeinst_as_edge(edge, empty_edges, owner, min_world, max_world) +codeinst_as_invoke_edge(interp::AbstractInterpreter, edge::MethodInstance) = codeinst_as_edge(interp, edge, empty_edges) # compute (and cache) an inferred AST and return the current best estimate of the result type @@ -749,6 +758,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize mi = specialize_method(method, atype, sparams) cache_mode = CACHE_MODE_GLOBAL # cache edge targets globally by default force_inline = is_stmt_inline(get_curr_ssaflag(caller)) + edge_ci = nothing let codeinst = get(code_cache(interp), mi, nothing) if codeinst isa CodeInstance # return existing rettype if the code is already inferred inferred = @atomic :monotonic codeinst.inferred @@ -757,6 +767,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize # nevertheless we re-infer it here again in order to propagate the re-inferred # source to the inliner as a volatile result cache_mode = CACHE_MODE_VOLATILE + edge_ci = codeinst else @assert codeinst.def === mi "MethodInstance for cached edge does not match" return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) @@ -776,9 +787,9 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize end if frame === false # completely new, but check again after reserving in the engine - edge_ci = ci_from_engine = cache_mode == CACHE_MODE_GLOBAL ? - engine_reserve(interp, mi) : nothing - if ci_from_engine !== nothing + if cache_mode == CACHE_MODE_GLOBAL + ci_from_engine = engine_reserve(interp, mi) + edge_ci = ci_from_engine codeinst = get(code_cache(interp), mi, nothing) if codeinst isa CodeInstance # return existing rettype if the code is already inferred engine_reject(interp, ci_from_engine) @@ -792,6 +803,8 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) end end + else + ci_from_engine = nothing end result = InferenceResult(mi, typeinf_lattice(interp)) if ci_from_engine !== nothing @@ -820,7 +833,10 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize local exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) # propagate newly inferred source to the inliner, allowing efficient inlining w/o deserialization: # note that this result is cached globally exclusively, so we can use this local result destructively - local volatile_inf_result = isinferred ? VolatileInferenceResult(result) : nothing + local volatile_inf_result = if isinferred && edge_ci isa CodeInstance + result.ci_as_edge = edge_ci # set the edge for the inliner usage + VolatileInferenceResult(result) + else nothing end mresult[] = MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, edge, edgecycle, edgelimited, volatile_inf_result) return true diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index 3c286aa742891..acd42b7a52165 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -264,9 +264,9 @@ function iterate(iter::BackedgeIterator, i::Int=1) i > length(backedges) && return nothing item = backedges[i] item isa Int && (i += 2; continue) # ignore the query information if present - isa(item, MethodInstance) && return BackedgePair(nothing, item), i+1 # regular dispatch + isa(item, CodeInstance) && return BackedgePair(nothing, item.def), i+1 # regular dispatch isa(item, MethodTable) && return BackedgePair(backedges[i+1], item), i+2 # abstract dispatch - return BackedgePair(item, backedges[i+1]::MethodInstance), i+2 # `invoke` calls + return BackedgePair(item, (backedges[i+1]::CodeInstance).def), i+2 # `invoke` calls end end diff --git a/test/compiler/contextual.jl b/test/compiler/contextual.jl index 408f01b1aea6a..c66384237f423 100644 --- a/test/compiler/contextual.jl +++ b/test/compiler/contextual.jl @@ -89,7 +89,8 @@ module MiniCassette @assert isa(src, CodeInfo) src = copy(src) @assert src.edges === Core.svec() - src.edges = MethodInstance[mi] + self_edge = Core.Compiler.codeinst_as_invoke_edge(mi, #=owner=#nothing, world, world) + src.edges = CodeInstance[self_edge] transform!(mi, src, length(args), match.sparams) # TODO: this is mandatory: code_info.min_world = max(code_info.min_world, min_world[]) # TODO: this is mandatory: code_info.max_world = min(code_info.max_world, max_world[])