Skip to content

Commit

Permalink
add edges metadata field to CodeInfo/CodeInstance, generate backedges…
Browse files Browse the repository at this point in the history 
… from it

This records all edge metadata in a field of any CI objects so that
staticdata can directly validate those objects from just that data.

Co-authored-by: Shuhei Kadowaki <[email protected]>
  • Loading branch information
@vtjnash @aviatesk
vtjnash and aviatesk committed Oct 31, 2024
1 parent da74ef1 commit 809d3fb
Show file tree
Hide file tree
Showing 38 changed files with 1,159 additions and 1,222 deletions.
1 change: 0 additions & 1 deletion base/Base.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ function Core._hasmethod(@nospecialize(f), @nospecialize(t)) # this function has
return Core._hasmethod(tt)
end


# core operations & types
include("promotion.jl")
include("tuple.jl")
Expand Down
6 changes: 3 additions & 3 deletions base/boot.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,11 +535,11 @@ function CodeInstance(
mi::MethodInstance, owner, @nospecialize(rettype), @nospecialize(exctype), @nospecialize(inferred_const),
@nospecialize(inferred), const_flags::Int32, min_world::UInt, max_world::UInt,
effects::UInt32, @nospecialize(analysis_results),
relocatability::UInt8, edges::Union{DebugInfo,Nothing})
relocatability::UInt8, di::Union{DebugInfo,Nothing}, edges::SimpleVector)
return ccall(:jl_new_codeinst, Ref{CodeInstance},
(Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any),
(Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any),
mi, owner, rettype, exctype, inferred_const, inferred, const_flags, min_world, max_world,
effects, analysis_results, relocatability, edges)
effects, analysis_results, relocatability, di, edges)
end
GlobalRef(m::Module, s::Symbol) = ccall(:jl_module_globalref, Ref{GlobalRef}, (Any, Any), m, s)
Module(name::Symbol=:anonymous, std_imports::Bool=true, default_names::Bool=true) = ccall(:jl_f_new_module, Ref{Module}, (Any, Bool, Bool), name, std_imports, default_names)
Expand Down
215 changes: 96 additions & 119 deletions base/compiler/abstractinterpretation.jl
View file

Large diffs are not rendered by default.

62 changes: 7 additions & 55 deletions base/compiler/inferencestate.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ mutable struct InferenceState
# TODO: Could keep this sparsely by doing structural liveness analysis ahead of time.
bb_vartables::Vector{Union{Nothing,VarTable}} # nothing if not analyzed yet
ssavaluetypes::Vector{Any}
stmt_edges::Vector{Vector{Any}}
edges::Vector{Any}
stmt_info::Vector{CallInfo}

#= intermediate states for interprocedural abstract interpretation =#
Expand Down Expand Up @@ -302,7 +302,7 @@ mutable struct InferenceState
nssavalues = src.ssavaluetypes::Int
ssavalue_uses = find_ssavalue_uses(code, nssavalues)
nstmts = length(code)
stmt_edges = Vector{Vector{Any}}(undef, nstmts)
edges = []
stmt_info = CallInfo[ NoCallInfo() for i = 1:nstmts ]

nslots = length(src.slotflags)
Expand All @@ -327,7 +327,7 @@ mutable struct InferenceState
unreachable = BitSet()
pclimitations = IdSet{InferenceState}()
limitations = IdSet{InferenceState}()
cycle_backedges = Vector{Tuple{InferenceState,Int}}()
cycle_backedges = Tuple{InferenceState,Int}[]
callstack = AbsIntState[]
tasks = WorkThunk[]

Expand All @@ -350,10 +350,12 @@ mutable struct InferenceState

restrict_abstract_call_sites = isa(def, Module)

parentid = frameid = cycleid = 0

this = new(
mi, world, mod, sptypes, slottypes, src, cfg, spec_info,
currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, stmt_edges, stmt_info,
tasks, pclimitations, limitations, cycle_backedges, callstack, 0, 0, 0,
currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, edges, stmt_info,
tasks, pclimitations, limitations, cycle_backedges, callstack, parentid, frameid, cycleid,
result, unreachable, valid_worlds, bestguess, exc_bestguess, ipo_effects,
restrict_abstract_call_sites, cache_mode, insert_coverage,
interp)
Expand Down Expand Up @@ -754,30 +756,6 @@ function record_ssa_assign!(𝕃ᵢ::AbstractLattice, ssa_id::Int, @nospecialize
return nothing
end

function add_cycle_backedge!(caller::InferenceState, frame::InferenceState)
update_valid_age!(caller, frame.valid_worlds)
backedge = (caller, caller.currpc)
contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge)
add_backedge!(caller, frame.linfo)
return frame
end

function get_stmt_edges!(caller::InferenceState, currpc::Int=caller.currpc)
stmt_edges = caller.stmt_edges
if !isassigned(stmt_edges, currpc)
return stmt_edges[currpc] = Any[]
else
return stmt_edges[currpc]
end
end

function empty_backedges!(frame::InferenceState, currpc::Int=frame.currpc)
if isassigned(frame.stmt_edges, currpc)
empty!(frame.stmt_edges[currpc])
end
return nothing
end

function narguments(sv::InferenceState, include_va::Bool=true)
nargs = Int(sv.src.nargs)
if !include_va
Expand Down Expand Up @@ -1008,32 +986,6 @@ function callers_in_cycle(sv::InferenceState)
end
callers_in_cycle(sv::IRInterpretationState) = AbsIntCycle(sv.callstack::Vector{AbsIntState}, 0, 0)

# temporarily accumulate our edges to later add as backedges in the callee
function add_backedge!(caller::InferenceState, mi::MethodInstance)
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
return push!(get_stmt_edges!(caller), mi)
end
function add_backedge!(irsv::IRInterpretationState, mi::MethodInstance)
return push!(irsv.edges, mi)
end

function add_invoke_backedge!(caller::InferenceState, @nospecialize(invokesig::Type), mi::MethodInstance)
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
return push!(get_stmt_edges!(caller), invokesig, mi)
end
function add_invoke_backedge!(irsv::IRInterpretationState, @nospecialize(invokesig::Type), mi::MethodInstance)
return push!(irsv.edges, invokesig, mi)
end

# used to temporarily accumulate our no method errors to later add as backedges in the callee method table
function add_mt_backedge!(caller::InferenceState, mt::MethodTable, @nospecialize(typ))
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
return push!(get_stmt_edges!(caller), mt, typ)
end
function add_mt_backedge!(irsv::IRInterpretationState, mt::MethodTable, @nospecialize(typ))
return push!(irsv.edges, mt, typ)
end

get_curr_ssaflag(sv::InferenceState) = sv.src.ssaflags[sv.currpc]
get_curr_ssaflag(sv::IRInterpretationState) = sv.ir.stmts[sv.curridx][:flag]

Expand Down
4 changes: 2 additions & 2 deletions base/compiler/optimize.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,7 @@ struct InliningState{Interp<:AbstractInterpreter}
interp::Interp
end
function InliningState(sv::InferenceState, interp::AbstractInterpreter)
edges = sv.stmt_edges[1]
return InliningState(edges, sv.world, interp)
return InliningState(sv.edges, sv.world, interp)
end
function InliningState(interp::AbstractInterpreter)
return InliningState(Any[], get_inference_world(interp), interp)
Expand Down Expand Up @@ -225,6 +224,7 @@ include("compiler/ssair/irinterp.jl")
function ir_to_codeinf!(opt::OptimizationState)
(; linfo, src) = opt
src = ir_to_codeinf!(src, opt.ir::IRCode)
src.edges = opt.inlining.edges
opt.ir = nothing
maybe_validate_code(linfo, src, "optimized")
return src
Expand Down
92 changes: 45 additions & 47 deletions base/compiler/ssair/inlining.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ end

struct ConstantCase
val::Any
ConstantCase(@nospecialize val) = new(val)
edge::CodeInstance
ConstantCase(@nospecialize(val), edge::CodeInstance) = new(val, edge)
end

struct SomeCase
Expand Down Expand Up @@ -68,11 +69,12 @@ struct InliningEdgeTracker
new(state.edges, invokesig)
end

function add_inlining_backedge!((; edges, invokesig)::InliningEdgeTracker, mi::MethodInstance)
function add_inlining_edge!(et::InliningEdgeTracker, edge::Union{CodeInstance,MethodInstance})
(; edges, invokesig) = et
if invokesig === nothing
push!(edges, mi)
add_one_edge!(edges, edge)
else # invoke backedge
push!(edges, invoke_signature(invokesig), mi)
add_invoke_edge!(edges, invoke_signature(invokesig), edge)
end
return nothing
end
Expand Down Expand Up @@ -784,10 +786,10 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}},
end

function compileable_specialization(mi::MethodInstance, effects::Effects,
et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true)
et::InliningEdgeTracker, @nospecialize(info::CallInfo), state::InliningState)
mi_invoke = mi
method, atype, sparams = mi.def::Method, mi.specTypes, mi.sparam_vals
if compilesig_invokes
if OptimizationParams(state.interp).compilesig_invokes
new_atype = get_compileable_sig(method, atype, sparams)
new_atype === nothing && return nothing
if atype !== new_atype
Expand All @@ -805,43 +807,41 @@ function compileable_specialization(mi::MethodInstance, effects::Effects,
return nothing
end
end
add_inlining_backedge!(et, mi) # to the dispatch lookup
mi_invoke !== mi && push!(et.edges, method.sig, mi_invoke) # add_inlining_backedge to the invoke call, if that is different
add_inlining_edge!(et, mi) # to the dispatch lookup
if mi_invoke !== mi
add_invoke_edge!(et.edges, method.sig, mi_invoke) # add_inlining_edge to the invoke call, if that is different
end
return InvokeCase(mi_invoke, effects, info)
end

function compileable_specialization(match::MethodMatch, effects::Effects,
et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true)
mi = specialize_method(match)
return compileable_specialization(mi, effects, et, info; compilesig_invokes)
end

struct InferredResult
src::Any # CodeInfo or IRCode
effects::Effects
InferredResult(@nospecialize(src), effects::Effects) = new(src, effects)
edge::CodeInstance
InferredResult(@nospecialize(src), effects::Effects, edge::CodeInstance) = new(src, effects, edge)
end
@inline function get_cached_result(state::InliningState, mi::MethodInstance)
code = get(code_cache(state), mi, nothing)
if code isa CodeInstance
if use_const_api(code)
# in this case function can be inlined to a constant
return ConstantCase(quoted(code.rettype_const))
return ConstantCase(quoted(code.rettype_const), code)
end
return code
end
return nothing
end
@inline function get_local_result(inf_result::InferenceResult)
@assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge"
effects = inf_result.ipo_effects
if is_foldable_nothrow(effects)
res = inf_result.result
if isa(res, Const) && is_inlineable_constant(res.val)
# use constant calling convention
return ConstantCase(quoted(res.val))
return ConstantCase(quoted(res.val), inf_result.ci_as_edge)
end
end
return InferredResult(inf_result.src, effects)
return InferredResult(inf_result.src, effects, inf_result.ci_as_edge)
end

# the general resolver for usual and const-prop'ed calls
Expand All @@ -861,30 +861,28 @@ function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult,
inferred_result = get_cached_result(state, mi)
end
if inferred_result isa ConstantCase
add_inlining_backedge!(et, mi)
add_inlining_edge!(et, inferred_result.edge)
return inferred_result
elseif inferred_result isa InferredResult
(; src, effects) = inferred_result
(; src, effects, edge) = inferred_result
elseif inferred_result isa CodeInstance
src = @atomic :monotonic inferred_result.inferred
effects = decode_effects(inferred_result.ipo_purity_bits)
edge = inferred_result
else # there is no cached source available, bail out
return compileable_specialization(mi, Effects(), et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(mi, Effects(), et, info, state)
end

# the duplicated check might have been done already within `analyze_method!`, but still
# we need it here too since we may come here directly using a constant-prop' result
if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag)
return compileable_specialization(mi, effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(edge.def, effects, et, info, state)
end

src_inlining_policy(state.interp, src, info, flag) ||
return compileable_specialization(mi, effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(edge.def, effects, et, info, state)

add_inlining_backedge!(et, mi)
add_inlining_edge!(et, edge)
if inferred_result isa CodeInstance
ir, spec_info, debuginfo = retrieve_ir_for_inlining(inferred_result, src)
else
Expand All @@ -904,7 +902,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U

cached_result = get_cached_result(state, mi)
if cached_result isa ConstantCase
add_inlining_backedge!(et, mi)
add_inlining_edge!(et, cached_result.edge)
return cached_result
elseif cached_result isa CodeInstance
src = @atomic :monotonic cached_result.inferred
Expand All @@ -915,7 +913,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U

src_inlining_policy(state.interp, src, info, flag) || return nothing
ir, spec_info, debuginfo = retrieve_ir_for_inlining(cached_result, src)
add_inlining_backedge!(et, mi)
add_inlining_edge!(et, cached_result)
return InliningTodo(mi, ir, spec_info, debuginfo, effects)
end

Expand Down Expand Up @@ -1119,7 +1117,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
# e.g. rewrite `((t::Tuple)...,)` to `t`
nonempty_idx = 0
𝕃ₒ = optimizer_lattice(state.interp)
for i = (arg_start + 1):length(argtypes)
for i = (arg_start+1):length(argtypes)
ti = argtypes[i]
⊑(𝕃ₒ, ti, Tuple{}) && continue
if ⊑(𝕃ₒ, ti, Tuple) && nonempty_idx == 0
Expand All @@ -1137,7 +1135,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
# Try to figure out the signature of the function being called
# and if rewrite_apply_exprargs can deal with this form
arginfos = MaybeAbstractIterationInfo[]
for i = (arg_start + 1):length(argtypes)
for i = (arg_start+1):length(argtypes)
thisarginfo = nothing
if !is_valid_type_for_apply_rewrite(argtypes[i], OptimizationParams(state.interp))
isa(info, ApplyCallInfo) || return nothing
Expand Down Expand Up @@ -1403,9 +1401,7 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig
result, match, argtypes, info, flag, state; allow_typevars=true)
end
if !fully_covered
atype = argtypes_to_type(sig.argtypes)
# We will emit an inline MethodError so we need a backedge to the MethodTable
add_uncovered_edges!(state.edges, info, atype)
# We will emit an inline MethodError in this case, but that info already came inference, so we must already have the uncovered edge for it
end
elseif !isempty(cases)
# if we've not seen all candidates, union split is valid only for dispatch tuples
Expand Down Expand Up @@ -1453,30 +1449,28 @@ end

function semiconcrete_result_item(result::SemiConcreteResult,
@nospecialize(info::CallInfo), flag::UInt32, state::InliningState)
mi = result.mi
mi = result.edge.def
et = InliningEdgeTracker(state)

if (!OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) ||
# For `NativeInterpreter`, `SemiConcreteResult` may be produced for
# a `@noinline`-declared method when it's marked as `@constprop :aggressive`.
# Suppress the inlining here (unless inlining is requested at the callsite).
(is_declared_noinline(mi.def::Method) && !is_stmt_inline(flag)))
return compileable_specialization(mi, result.effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(mi, result.effects, et, info, state)
end
src_inlining_policy(state.interp, result.ir, info, flag) ||
return compileable_specialization(mi, result.effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(mi, result.effects, et, info, state)

add_inlining_backedge!(et, mi)
add_inlining_edge!(et, result.edge)
preserve_local_sources = OptimizationParams(state.interp).preserve_local_sources
ir, _, debuginfo = retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources)
return InliningTodo(mi, ir, result.spec_info, debuginfo, result.effects)
end

function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult,
match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState)
mi = result.mi
mi = result.edge.def
validate_sparams(mi.sparam_vals) || return false
item = semiconcrete_result_item(result, info, flag, state)
item === nothing && return false
Expand All @@ -1499,11 +1493,10 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn
invokesig::Union{Nothing,Vector{Any}}=nothing)
if !may_inline_concrete_result(result)
et = InliningEdgeTracker(state, invokesig)
return compileable_specialization(result.mi, result.effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(result.edge.def, result.effects, et, info, state)
end
@assert result.effects === EFFECTS_TOTAL
return ConstantCase(quoted(result.result))
return ConstantCase(quoted(result.result), result.edge)
end

function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr,
Expand Down Expand Up @@ -1552,11 +1545,16 @@ function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOp
info isa MethodResultPure && (info = info.info)
info isa ConstCallInfo && (info = info.call)
info isa MethodMatchInfo || return nothing
length(info.results) == 1 || return nothing
length(info.edges) == length(info.results) == 1 || return nothing
match = info.results[1]::MethodMatch
match.fully_covers || return nothing
case = compileable_specialization(match, Effects(), InliningEdgeTracker(state), info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
edge = info.edges[1]
if edge === nothing
edge = specialize_method(match)
else
edge = edge.def
end
case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state)
case === nothing && return nothing
stmt.head = :invoke_modify
pushfirst!(stmt.args, case.invoke)
Expand Down
Loading

0 comments on commit 809d3fb

Please sign in to comment.