Skip to content

Commit

Permalink
Add edges vector to CodeInstance/CodeInfo to keep backedges as edges (
Browse files Browse the repository at this point in the history 
#54894)

Appears to add about 11MB (128MB to 139MB) to the system image, and to 
decrease the stdlib size by 55 MB (325MB to 270MB), so seems overall 
favorable right now. The edges are computed following the encoding 
<https://hackmd.io/sjPig55kS4a5XNWC6HmKSg?both#Edges-Encoding> to
correctly reflect the backedges.

Co-authored-by: Shuhei Kadowaki <[email protected]>
  • Loading branch information
@vtjnash @aviatesk
vtjnash and aviatesk authored Nov 1, 2024
1 parent 706a4f6 commit 7635190
Show file tree
Hide file tree
Showing 38 changed files with 1,159 additions and 1,222 deletions.
1 change: 0 additions & 1 deletion base/Base.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ function Core._hasmethod(@nospecialize(f), @nospecialize(t)) # this function has
return Core._hasmethod(tt)
end


# core operations & types
include("promotion.jl")
include("tuple.jl")
Expand Down
6 changes: 3 additions & 3 deletions base/boot.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,11 +535,11 @@ function CodeInstance(
mi::MethodInstance, owner, @nospecialize(rettype), @nospecialize(exctype), @nospecialize(inferred_const),
@nospecialize(inferred), const_flags::Int32, min_world::UInt, max_world::UInt,
effects::UInt32, @nospecialize(analysis_results),
relocatability::UInt8, edges::Union{DebugInfo,Nothing})
relocatability::UInt8, di::Union{DebugInfo,Nothing}, edges::SimpleVector)
return ccall(:jl_new_codeinst, Ref{CodeInstance},
(Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any),
(Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any),
mi, owner, rettype, exctype, inferred_const, inferred, const_flags, min_world, max_world,
effects, analysis_results, relocatability, edges)
effects, analysis_results, relocatability, di, edges)
end
GlobalRef(m::Module, s::Symbol) = ccall(:jl_module_globalref, Ref{GlobalRef}, (Any, Any), m, s)
Module(name::Symbol=:anonymous, std_imports::Bool=true, default_names::Bool=true) = ccall(:jl_f_new_module, Ref{Module}, (Any, Bool, Bool), name, std_imports, default_names)
Expand Down
215 changes: 96 additions & 119 deletions base/compiler/abstractinterpretation.jl
View file

Large diffs are not rendered by default.

62 changes: 7 additions & 55 deletions base/compiler/inferencestate.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ mutable struct InferenceState
# TODO: Could keep this sparsely by doing structural liveness analysis ahead of time.
bb_vartables::Vector{Union{Nothing,VarTable}} # nothing if not analyzed yet
ssavaluetypes::Vector{Any}
stmt_edges::Vector{Vector{Any}}
edges::Vector{Any}
stmt_info::Vector{CallInfo}

#= intermediate states for interprocedural abstract interpretation =#
Expand Down Expand Up @@ -302,7 +302,7 @@ mutable struct InferenceState
nssavalues = src.ssavaluetypes::Int
ssavalue_uses = find_ssavalue_uses(code, nssavalues)
nstmts = length(code)
stmt_edges = Vector{Vector{Any}}(undef, nstmts)
edges = []
stmt_info = CallInfo[ NoCallInfo() for i = 1:nstmts ]

nslots = length(src.slotflags)
Expand All @@ -327,7 +327,7 @@ mutable struct InferenceState
unreachable = BitSet()
pclimitations = IdSet{InferenceState}()
limitations = IdSet{InferenceState}()
cycle_backedges = Vector{Tuple{InferenceState,Int}}()
cycle_backedges = Tuple{InferenceState,Int}[]
callstack = AbsIntState[]
tasks = WorkThunk[]

Expand All @@ -350,10 +350,12 @@ mutable struct InferenceState

restrict_abstract_call_sites = isa(def, Module)

parentid = frameid = cycleid = 0

this = new(
mi, world, mod, sptypes, slottypes, src, cfg, spec_info,
currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, stmt_edges, stmt_info,
tasks, pclimitations, limitations, cycle_backedges, callstack, 0, 0, 0,
currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, edges, stmt_info,
tasks, pclimitations, limitations, cycle_backedges, callstack, parentid, frameid, cycleid,
result, unreachable, valid_worlds, bestguess, exc_bestguess, ipo_effects,
restrict_abstract_call_sites, cache_mode, insert_coverage,
interp)
Expand Down Expand Up @@ -754,30 +756,6 @@ function record_ssa_assign!(𝕃ᵢ::AbstractLattice, ssa_id::Int, @nospecialize
return nothing
end

function add_cycle_backedge!(caller::InferenceState, frame::InferenceState)
update_valid_age!(caller, frame.valid_worlds)
backedge = (caller, caller.currpc)
contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge)
add_backedge!(caller, frame.linfo)
return frame
end

function get_stmt_edges!(caller::InferenceState, currpc::Int=caller.currpc)
stmt_edges = caller.stmt_edges
if !isassigned(stmt_edges, currpc)
return stmt_edges[currpc] = Any[]
else
return stmt_edges[currpc]
end
end

function empty_backedges!(frame::InferenceState, currpc::Int=frame.currpc)
if isassigned(frame.stmt_edges, currpc)
empty!(frame.stmt_edges[currpc])
end
return nothing
end

function narguments(sv::InferenceState, include_va::Bool=true)
nargs = Int(sv.src.nargs)
if !include_va
Expand Down Expand Up @@ -1008,32 +986,6 @@ function callers_in_cycle(sv::InferenceState)
end
callers_in_cycle(sv::IRInterpretationState) = AbsIntCycle(sv.callstack::Vector{AbsIntState}, 0, 0)

# temporarily accumulate our edges to later add as backedges in the callee
function add_backedge!(caller::InferenceState, mi::MethodInstance)
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
return push!(get_stmt_edges!(caller), mi)
end
function add_backedge!(irsv::IRInterpretationState, mi::MethodInstance)
return push!(irsv.edges, mi)
end

function add_invoke_backedge!(caller::InferenceState, @nospecialize(invokesig::Type), mi::MethodInstance)
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
return push!(get_stmt_edges!(caller), invokesig, mi)
end
function add_invoke_backedge!(irsv::IRInterpretationState, @nospecialize(invokesig::Type), mi::MethodInstance)
return push!(irsv.edges, invokesig, mi)
end

# used to temporarily accumulate our no method errors to later add as backedges in the callee method table
function add_mt_backedge!(caller::InferenceState, mt::MethodTable, @nospecialize(typ))
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
return push!(get_stmt_edges!(caller), mt, typ)
end
function add_mt_backedge!(irsv::IRInterpretationState, mt::MethodTable, @nospecialize(typ))
return push!(irsv.edges, mt, typ)
end

get_curr_ssaflag(sv::InferenceState) = sv.src.ssaflags[sv.currpc]
get_curr_ssaflag(sv::IRInterpretationState) = sv.ir.stmts[sv.curridx][:flag]

Expand Down
4 changes: 2 additions & 2 deletions base/compiler/optimize.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,7 @@ struct InliningState{Interp<:AbstractInterpreter}
interp::Interp
end
function InliningState(sv::InferenceState, interp::AbstractInterpreter)
edges = sv.stmt_edges[1]
return InliningState(edges, sv.world, interp)
return InliningState(sv.edges, sv.world, interp)
end
function InliningState(interp::AbstractInterpreter)
return InliningState(Any[], get_inference_world(interp), interp)
Expand Down Expand Up @@ -225,6 +224,7 @@ include("compiler/ssair/irinterp.jl")
function ir_to_codeinf!(opt::OptimizationState)
(; linfo, src) = opt
src = ir_to_codeinf!(src, opt.ir::IRCode)
src.edges = opt.inlining.edges
opt.ir = nothing
maybe_validate_code(linfo, src, "optimized")
return src
Expand Down
92 changes: 45 additions & 47 deletions base/compiler/ssair/inlining.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ end

struct ConstantCase
val::Any
ConstantCase(@nospecialize val) = new(val)
edge::CodeInstance
ConstantCase(@nospecialize(val), edge::CodeInstance) = new(val, edge)
end

struct SomeCase
Expand Down Expand Up @@ -68,11 +69,12 @@ struct InliningEdgeTracker
new(state.edges, invokesig)
end

function add_inlining_backedge!((; edges, invokesig)::InliningEdgeTracker, mi::MethodInstance)
function add_inlining_edge!(et::InliningEdgeTracker, edge::Union{CodeInstance,MethodInstance})
(; edges, invokesig) = et
if invokesig === nothing
push!(edges, mi)
add_one_edge!(edges, edge)
else # invoke backedge
push!(edges, invoke_signature(invokesig), mi)
add_invoke_edge!(edges, invoke_signature(invokesig), edge)
end
return nothing
end
Expand Down Expand Up @@ -784,10 +786,10 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}},
end

function compileable_specialization(mi::MethodInstance, effects::Effects,
et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true)
et::InliningEdgeTracker, @nospecialize(info::CallInfo), state::InliningState)
mi_invoke = mi
method, atype, sparams = mi.def::Method, mi.specTypes, mi.sparam_vals
if compilesig_invokes
if OptimizationParams(state.interp).compilesig_invokes
new_atype = get_compileable_sig(method, atype, sparams)
new_atype === nothing && return nothing
if atype !== new_atype
Expand All @@ -805,43 +807,41 @@ function compileable_specialization(mi::MethodInstance, effects::Effects,
return nothing
end
end
add_inlining_backedge!(et, mi) # to the dispatch lookup
mi_invoke !== mi && push!(et.edges, method.sig, mi_invoke) # add_inlining_backedge to the invoke call, if that is different
add_inlining_edge!(et, mi) # to the dispatch lookup
if mi_invoke !== mi
add_invoke_edge!(et.edges, method.sig, mi_invoke) # add_inlining_edge to the invoke call, if that is different
end
return InvokeCase(mi_invoke, effects, info)
end

function compileable_specialization(match::MethodMatch, effects::Effects,
et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true)
mi = specialize_method(match)
return compileable_specialization(mi, effects, et, info; compilesig_invokes)
end

struct InferredResult
src::Any # CodeInfo or IRCode
effects::Effects
InferredResult(@nospecialize(src), effects::Effects) = new(src, effects)
edge::CodeInstance
InferredResult(@nospecialize(src), effects::Effects, edge::CodeInstance) = new(src, effects, edge)
end
@inline function get_cached_result(state::InliningState, mi::MethodInstance)
code = get(code_cache(state), mi, nothing)
if code isa CodeInstance
if use_const_api(code)
# in this case function can be inlined to a constant
return ConstantCase(quoted(code.rettype_const))
return ConstantCase(quoted(code.rettype_const), code)
end
return code
end
return nothing
end
@inline function get_local_result(inf_result::InferenceResult)
@assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge"
effects = inf_result.ipo_effects
if is_foldable_nothrow(effects)
res = inf_result.result
if isa(res, Const) && is_inlineable_constant(res.val)
# use constant calling convention
return ConstantCase(quoted(res.val))
return ConstantCase(quoted(res.val), inf_result.ci_as_edge)
end
end
return InferredResult(inf_result.src, effects)
return InferredResult(inf_result.src, effects, inf_result.ci_as_edge)
end

# the general resolver for usual and const-prop'ed calls
Expand All @@ -861,30 +861,28 @@ function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult,
inferred_result = get_cached_result(state, mi)
end
if inferred_result isa ConstantCase
add_inlining_backedge!(et, mi)
add_inlining_edge!(et, inferred_result.edge)
return inferred_result
elseif inferred_result isa InferredResult
(; src, effects) = inferred_result
(; src, effects, edge) = inferred_result
elseif inferred_result isa CodeInstance
src = @atomic :monotonic inferred_result.inferred
effects = decode_effects(inferred_result.ipo_purity_bits)
edge = inferred_result
else # there is no cached source available, bail out
return compileable_specialization(mi, Effects(), et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(mi, Effects(), et, info, state)
end

# the duplicated check might have been done already within `analyze_method!`, but still
# we need it here too since we may come here directly using a constant-prop' result
if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag)
return compileable_specialization(mi, effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(edge.def, effects, et, info, state)
end

src_inlining_policy(state.interp, src, info, flag) ||
return compileable_specialization(mi, effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(edge.def, effects, et, info, state)

add_inlining_backedge!(et, mi)
add_inlining_edge!(et, edge)
if inferred_result isa CodeInstance
ir, spec_info, debuginfo = retrieve_ir_for_inlining(inferred_result, src)
else
Expand All @@ -904,7 +902,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U

cached_result = get_cached_result(state, mi)
if cached_result isa ConstantCase
add_inlining_backedge!(et, mi)
add_inlining_edge!(et, cached_result.edge)
return cached_result
elseif cached_result isa CodeInstance
src = @atomic :monotonic cached_result.inferred
Expand All @@ -915,7 +913,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U

src_inlining_policy(state.interp, src, info, flag) || return nothing
ir, spec_info, debuginfo = retrieve_ir_for_inlining(cached_result, src)
add_inlining_backedge!(et, mi)
add_inlining_edge!(et, cached_result)
return InliningTodo(mi, ir, spec_info, debuginfo, effects)
end

Expand Down Expand Up @@ -1119,7 +1117,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
# e.g. rewrite `((t::Tuple)...,)` to `t`
nonempty_idx = 0
𝕃ₒ = optimizer_lattice(state.interp)
for i = (arg_start + 1):length(argtypes)
for i = (arg_start+1):length(argtypes)
ti = argtypes[i]
⊑(𝕃ₒ, ti, Tuple{}) && continue
if ⊑(𝕃ₒ, ti, Tuple) && nonempty_idx == 0
Expand All @@ -1137,7 +1135,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
# Try to figure out the signature of the function being called
# and if rewrite_apply_exprargs can deal with this form
arginfos = MaybeAbstractIterationInfo[]
for i = (arg_start + 1):length(argtypes)
for i = (arg_start+1):length(argtypes)
thisarginfo = nothing
if !is_valid_type_for_apply_rewrite(argtypes[i], OptimizationParams(state.interp))
isa(info, ApplyCallInfo) || return nothing
Expand Down Expand Up @@ -1403,9 +1401,7 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig
result, match, argtypes, info, flag, state; allow_typevars=true)
end
if !fully_covered
atype = argtypes_to_type(sig.argtypes)
# We will emit an inline MethodError so we need a backedge to the MethodTable
add_uncovered_edges!(state.edges, info, atype)
# We will emit an inline MethodError in this case, but that info already came inference, so we must already have the uncovered edge for it
end
elseif !isempty(cases)
# if we've not seen all candidates, union split is valid only for dispatch tuples
Expand Down Expand Up @@ -1453,30 +1449,28 @@ end

function semiconcrete_result_item(result::SemiConcreteResult,
@nospecialize(info::CallInfo), flag::UInt32, state::InliningState)
mi = result.mi
mi = result.edge.def
et = InliningEdgeTracker(state)

if (!OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) ||
# For `NativeInterpreter`, `SemiConcreteResult` may be produced for
# a `@noinline`-declared method when it's marked as `@constprop :aggressive`.
# Suppress the inlining here (unless inlining is requested at the callsite).
(is_declared_noinline(mi.def::Method) && !is_stmt_inline(flag)))
return compileable_specialization(mi, result.effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(mi, result.effects, et, info, state)
end
src_inlining_policy(state.interp, result.ir, info, flag) ||
return compileable_specialization(mi, result.effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(mi, result.effects, et, info, state)

add_inlining_backedge!(et, mi)
add_inlining_edge!(et, result.edge)
preserve_local_sources = OptimizationParams(state.interp).preserve_local_sources
ir, _, debuginfo = retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources)
return InliningTodo(mi, ir, result.spec_info, debuginfo, result.effects)
end

function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult,
match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState)
mi = result.mi
mi = result.edge.def
validate_sparams(mi.sparam_vals) || return false
item = semiconcrete_result_item(result, info, flag, state)
item === nothing && return false
Expand All @@ -1499,11 +1493,10 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn
invokesig::Union{Nothing,Vector{Any}}=nothing)
if !may_inline_concrete_result(result)
et = InliningEdgeTracker(state, invokesig)
return compileable_specialization(result.mi, result.effects, et, info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
return compileable_specialization(result.edge.def, result.effects, et, info, state)
end
@assert result.effects === EFFECTS_TOTAL
return ConstantCase(quoted(result.result))
return ConstantCase(quoted(result.result), result.edge)
end

function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr,
Expand Down Expand Up @@ -1552,11 +1545,16 @@ function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOp
info isa MethodResultPure && (info = info.info)
info isa ConstCallInfo && (info = info.call)
info isa MethodMatchInfo || return nothing
length(info.results) == 1 || return nothing
length(info.edges) == length(info.results) == 1 || return nothing
match = info.results[1]::MethodMatch
match.fully_covers || return nothing
case = compileable_specialization(match, Effects(), InliningEdgeTracker(state), info;
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
edge = info.edges[1]
if edge === nothing
edge = specialize_method(match)
else
edge = edge.def
end
case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state)
case === nothing && return nothing
stmt.head = :invoke_modify
pushfirst!(stmt.args, case.invoke)
Expand Down
Loading

2 comments on commit 7635190

@vtjnash
Copy link
Member Author

@vtjnash vtjnash commented on 7635190 Nov 1, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nanosoldier runbenchmarks(ALL, isdaily = true)

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your benchmark job has completed - possible performance regressions were detected. A full report can be found here.

Please sign in to comment.