override edges with const-prop'ed edges

base/compiler/abstractinterpretation.jl

@@ -106,9 +106,9 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f),
                     mresult[], f.contents, this_arginfo, si, match, sv)
                 const_result = volatile_inf_result
                 if const_call_result !== nothing
-                    # TODO override the edge with the const-prop' edge
                     this_const_conditional = ignorelimited(const_call_result.rt)
                     this_const_rt = widenwrappedconditional(const_call_result.rt)
+                    const_edge = nothing
                     if this_const_rt ⊑ₚ this_rt
                         # As long as the const-prop result we have is not *worse* than
                         # what we found out on types, we'd like to use it. Even if the
@@ -119,20 +119,23 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f),
                         # e.g. in cases when there are cycles but cached result is still accurate
                         this_conditional = this_const_conditional
                         this_rt = this_const_rt
-                        (; effects, const_result) = const_call_result
+                        (; effects, const_result, const_edge) = const_call_result
                     elseif is_better_effects(const_call_result.effects, effects)
-                        (; effects, const_result) = const_call_result
+                        (; effects, const_result, const_edge) = const_call_result
                     else
                         add_remark!(interp, sv, "[constprop] Discarded because the result was wider than inference")
                     end
                     # Treat the exception type separately. Currently, constprop often cannot determine the exception type
                     # because consistent-cy does not apply to exceptions.
                     if const_call_result.exct ⋤ this_exct
                         this_exct = const_call_result.exct
-                        (; const_result) = const_call_result
+                        (; const_result, const_edge) = const_call_result
                     else
                         add_remark!(interp, sv, "[constprop] Discarded exception type because result was wider than inference")
                     end
+                    if const_edge !== nothing
+                        edge = const_edge
+                    end
                 end
 
                 all_effects = merge_effects(all_effects, effects)
@@ -820,11 +823,12 @@ struct ConstCallResult
     exct::Any
     const_result::ConstResult
     effects::Effects
+    const_edge::Union{Nothing,CodeInstance}
     function ConstCallResult(
         @nospecialize(rt), @nospecialize(exct),
-        const_result::ConstResult,
-        effects::Effects)
-        return new(rt, exct, const_result, effects)
+        const_result::ConstResult, effects::Effects,
+        const_edge::Union{Nothing,CodeInstance})
+        return new(rt, exct, const_result, effects, const_edge)
     end
 end
 
@@ -975,9 +979,11 @@ function concrete_eval_call(interp::AbstractInterpreter,
     catch e
         # The evaluation threw. By :consistent-cy, we're guaranteed this would have happened at runtime.
         # Howevever, at present, :consistency does not mandate the type of the exception
-        return ConstCallResult(Bottom, Any, ConcreteResult(edge, result.effects), result.effects)
+        concrete_result = ConcreteResult(edge, result.effects)
+        return ConstCallResult(Bottom, Any, concrete_result, result.effects, #=const_edge=#nothing)
     end
-    return ConstCallResult(Const(value), Union{}, ConcreteResult(edge, EFFECTS_TOTAL, value), EFFECTS_TOTAL)
+    concrete_result = ConcreteResult(edge, EFFECTS_TOTAL, value)
+    return ConstCallResult(Const(value), Bottom, concrete_result, EFFECTS_TOTAL, #=const_edge=#nothing)
 end
 
 # check if there is a cycle and duplicated inference of `mi`
@@ -1242,16 +1248,21 @@ function semi_concrete_eval_call(interp::AbstractInterpreter,
                     effects = Effects(effects; noub=ALWAYS_TRUE)
                 end
                 exct = refine_exception_type(result.exct, effects)
-                return ConstCallResult(rt, exct, SemiConcreteResult(codeinst, ir, effects, spec_info(irsv)), effects)
+                semi_concrete_result = SemiConcreteResult(codeinst, ir, effects, spec_info(irsv))
+                const_edge = nothing # TODO use the edges from irsv?
+                return ConstCallResult(rt, exct, semi_concrete_result, effects, const_edge)
             end
         end
     end
     return nothing
 end
 
-const_prop_result(inf_result::InferenceResult) =
-    ConstCallResult(inf_result.result, inf_result.exc_result, ConstPropResult(inf_result),
-                    inf_result.ipo_effects)
+function const_prop_result(inf_result::InferenceResult)
+    @assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge"
+    const_prop_result = ConstPropResult(inf_result)
+    return ConstCallResult(inf_result.result, inf_result.exc_result, const_prop_result,
+                           inf_result.ipo_effects, inf_result.ci_as_edge)
+end
 
 # return cached result of constant analysis
 return_localcache_result(::AbstractInterpreter, inf_result::InferenceResult, ::AbsIntState) =
@@ -1264,7 +1275,7 @@ end
 
 function const_prop_call(interp::AbstractInterpreter,
     mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState,
-    concrete_eval_result::Union{Nothing, ConstCallResult}=nothing)
+    concrete_eval_result::Union{Nothing,ConstCallResult}=nothing)
     inf_cache = get_inference_cache(interp)
     𝕃ᵢ = typeinf_lattice(interp)
     forwarded_argtypes = compute_forwarded_argtypes(interp, arginfo, sv)
@@ -1312,6 +1323,7 @@ function const_prop_call(interp::AbstractInterpreter,
         pop!(callstack)
         return nothing
     end
+    inf_result.ci_as_edge = codeinst_as_edge(interp, mi, Core.svec(frame.edges...))
     @assert frame.frameid != 0 && frame.cycleid == frame.frameid
     @assert frame.parentid == sv.frameid
     @assert inf_result.result !== nothing
@@ -2209,12 +2221,15 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt
             result, f, arginfo, si, match, sv, invokecall)
         const_result = volatile_inf_result
         if const_call_result !== nothing
-            # TODO override the edge with the const-prop' edge
+            const_edge = nothing
             if const_call_result.rt ⊑ rt
-                (; rt, effects, const_result) = const_call_result
+                (; rt, effects, const_result, const_edge) = const_call_result
             end
             if const_call_result.exct ⋤ exct
-                (; exct, const_result) = const_call_result
+                (; exct, const_result, const_edge) = const_call_result
+            end
+            if const_edge !== nothing
+                edge = const_edge
             end
         end
         rt = from_interprocedural!(interp, rt, sv, arginfo, sig)
@@ -2423,12 +2438,15 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter,
             const_call_result = abstract_call_method_with_const_args(interp, result,
                 #=f=#nothing, arginfo, si, match, sv)
             if const_call_result !== nothing
-                # TODO override the edge with the const-prop' edge
+                const_edge = nothing
                 if const_call_result.rt ⊑ rt
-                    (; rt, effects, const_result) = const_call_result
+                    (; rt, effects, const_result, const_edge) = const_call_result
                 end
                 if const_call_result.exct ⋤ exct
-                    (; exct, const_result) = const_call_result
+                    (; exct, const_result, const_edge) = const_call_result
+                end
+                if const_edge !== nothing
+                    edge = const_edge
                 end
             end
         end

base/compiler/tfuncs.jl

@@ -3012,7 +3012,7 @@ function abstract_applicable(interp::AbstractInterpreter, argtypes::Vector{Any},
         if rt !== Bool
             for i = 1:napplicable
                 (; match, edges, edge_idx) = applicable[i]
-                edges[edge_idx] = codeinstance_for_method_match_edge(interp, specialize_method(match))
+                edges[edge_idx] = codeinst_as_method_match_edge(interp, specialize_method(match))
             end
             info = VirtualMethodMatchInfo(matches.info)
         end
@@ -3055,20 +3055,13 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv
         vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge
     else
         rt = Const(true)
-        edge = codeinstance_for_method_match_edge(interp, specialize_method(match))
+        edge = codeinst_as_method_match_edge(interp, specialize_method(match))
         vinfo = InvokeCallInfo(edge, match, nothing, types)
     end
     info = VirtualMethodMatchInfo(vinfo)
     return CallMeta(rt, Union{}, EFFECTS_TOTAL, info)
 end
 
-# allocate a dummy `edge::CodeInstance` to be added by `add_edges!`
-function codeinstance_for_method_match_edge(interp::AbstractInterpreter, edge::MethodInstance)
-    return CodeInstance(edge, cache_owner(interp), Any, Any, nothing, nothing, zero(Int32),
-        get_inference_world(interp), get_inference_world(interp),
-        zero(UInt32), nothing, zero(UInt8), nothing, empty_edges)
-end
-
 # N.B.: typename maps type equivalence classes to a single value
 function typename_static(@nospecialize(t))
     t isa Const && return _typename(t.val)

base/compiler/typeinfer.jl

@@ -105,12 +105,9 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState;
         # but our caller might, so let's just make it anyways
         store_backedges(result, caller.edges)
     end
-    if isdefined(result, :ci)
-        ci = result.ci
+    if isdefined(result, :ci_for_cache)
+        ci = result.ci_for_cache
         inferred_result = nothing
-        # don't add backedges to toplevel method instance
-        istoplevel = caller.linfo.def isa Method
-        edges = istoplevel ? empty_edges : Core.svec(caller.edges...)
         relocatability = 0x1
         const_flag = is_result_constabi_eligible(result)
         if !can_discard_trees || (is_cached(caller) && !const_flag)
@@ -136,11 +133,13 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState;
         if !@isdefined di
             di = DebugInfo(result.linfo)
         end
+        istoplevel = caller.linfo.def isa Method # don't add backedges to toplevel method instance
+        edges = istoplevel ? empty_edges : Core.svec(caller.edges...)
+        min_world, max_world = first(valid_worlds), last(valid_worlds)
+        effects_bits = encode_effects(result.ipo_effects)
         ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any),
-                ci, inferred_result, const_flag,
-                first(result.valid_worlds), last(result.valid_worlds),
-                encode_effects(result.ipo_effects), result.analysis_results,
-                relocatability, di, edges)
+            ci, inferred_result, const_flag, min_world, max_world, effects_bits,
+            result.analysis_results, relocatability, di, edges)
         engine_reject(interp, ci)
     end
     return nothing
@@ -242,13 +241,13 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance
     end
 end
 
-function cache_result!(interp::AbstractInterpreter, result::InferenceResult)
+function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci_for_cache::CodeInstance)
     if last(result.valid_worlds) == get_world_counter()
         # if we've successfully recorded all of the backedges in the global reverse-cache,
         # we can now widen our applicability in the global cache too
         result.valid_worlds = WorldRange(first(result.valid_worlds), typemax(UInt))
     end
-    @assert isdefined(result.ci, :inferred)
+    @assert isdefined(ci_for_cache, :inferred)
     # check if the existing linfo metadata is also sufficient to describe the current inference result
     # to decide if it is worth caching this right now
     mi = result.linfo
@@ -259,7 +258,7 @@ function cache_result!(interp::AbstractInterpreter, result::InferenceResult)
         @assert isdefined(ci, :inferred)
         return false
     end
-    code_cache(interp)[mi] = result.ci
+    code_cache(interp)[mi] = ci_for_cache
     return true
 end
 
@@ -451,7 +450,7 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter)
     maybe_validate_code(me.linfo, me.src, "inferred")
 
     # finish populating inference results into the CodeInstance if possible, and maybe cache that globally for use elsewhere
-    if isdefined(result, :ci) && !limited_ret
+    if isdefined(result, :ci_for_cache) && !limited_ret
         result_type = result.result
         @assert !(result_type === nothing || result_type isa LimitedAccuracy)
         if isa(result_type, Const)
@@ -478,13 +477,17 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter)
         end
         relocatability = 0x0
         di = nothing
-        edges = istoplevel ? empty_edges : Core.svec(me.edges...)
+        edges = empty_edges # `edges` will be updated within `finish!`
+        ci_for_cache = result.ci_for_cache
+        rettype = widenconst(result_type)
+        exctype = widenconst(result.exc_result)
+        min_world, max_world = first(result.valid_worlds), last(result.valid_worlds)
+        effects_bits = encode_effects(result.ipo_effects)
         ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any),
-                result.ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags,
-                first(result.valid_worlds), last(result.valid_worlds),
-                encode_effects(result.ipo_effects), result.analysis_results, di, edges)
+            ci_for_cache, rettype, exctype, rettype_const, const_flags,
+            min_world, max_world, effects_bits, result.analysis_results, di, edges)
         if is_cached(me)
-            cached_result = cache_result!(me.interp, me.result)
+            cached_result = cache_result!(me.interp, result, ci_for_cache)
             if !cached_result
                 me.cache_mode = CACHE_MODE_NULL
             end
@@ -732,6 +735,15 @@ function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method
     return MethodCallResult(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result)
 end
 
+# allocate a dummy `edge::CodeInstance` to be added by `add_edges!`
+function codeinst_as_edge(interp::AbstractInterpreter, edge::MethodInstance, edges::SimpleVector)
+    return CodeInstance(edge, cache_owner(interp), Any, Any, nothing, nothing, zero(Int32),
+        get_inference_world(interp), get_inference_world(interp),
+        zero(UInt32), nothing, zero(UInt8), nothing, edges)
+end
+codeinst_as_method_match_edge(interp::AbstractInterpreter, edge::MethodInstance) =
+    codeinst_as_edge(interp, edge, empty_edges)
+
 # compute (and cache) an inferred AST and return the current best estimate of the result type
 function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool)
     mi = specialize_method(method, atype, sparams)
@@ -764,7 +776,8 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize
     end
     if frame === false
         # completely new, but check again after reserving in the engine
-        ci_from_engine = cache_mode == CACHE_MODE_GLOBAL ? engine_reserve(interp, mi) : nothing
+        edge_ci = ci_from_engine = cache_mode == CACHE_MODE_GLOBAL ?
+            engine_reserve(interp, mi) : nothing
         if ci_from_engine !== nothing
             codeinst = get(code_cache(interp), mi, nothing)
             if codeinst isa CodeInstance # return existing rettype if the code is already inferred
@@ -773,6 +786,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize
                 inferred = @atomic :monotonic codeinst.inferred
                 if inferred === nothing && force_inline
                     cache_mode = CACHE_MODE_VOLATILE
+                    edge_ci = codeinst
                 else
                     @assert codeinst.def === mi "MethodInstance for cached edge does not match"
                     return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited)
@@ -781,7 +795,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize
         end
         result = InferenceResult(mi, typeinf_lattice(interp))
         if ci_from_engine !== nothing
-            result.ci = ci_from_engine
+            result.ci_for_cache = ci_from_engine
         end
         frame = InferenceState(result, cache_mode, interp) # always use the cache for edge targets
         if frame === nothing
@@ -799,8 +813,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize
             push!(caller.tasks, function get_infer_result(interp, caller)
                 update_valid_age!(caller, frame.valid_worlds)
                 local isinferred = is_inferred(frame)
-                # TODO allocate an edge when isinferred && ci_from_engine === nothing (i.e. cache_mode === CACHE_MODE_LOCAL)
-                local edge = isinferred ? ci_from_engine : nothing
+                local edge = isinferred ? edge_ci : nothing
                 local effects = isinferred ? frame.result.ipo_effects : # effects are adjusted already within `finish` for ipo_effects
                     adjust_effects(effects_for_cycle(frame.ipo_effects), method)
                 local bestguess = frame.bestguess
@@ -1075,7 +1088,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod
         end
     end
     result = InferenceResult(mi, typeinf_lattice(interp))
-    result.ci = ci
+    result.ci_for_cache = ci
     frame = InferenceState(result, #=cache_mode=#:global, interp)
     if frame === nothing
         engine_reject(interp, ci)
@@ -1084,7 +1097,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod
     typeinf(interp, frame)
     ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time)
 
-    ci = result.ci # reload from result in case it changed
+    ci = result.ci_for_cache # reload from result in case it changed
     if source_mode == SOURCE_MODE_ABI && frame.cache_mode != CACHE_MODE_GLOBAL
         # XXX: jl_type_infer somewhat ambiguously assumes this must be cached, while jl_ci_cache_lookup sort of ambiguously re-caches it
         # XXX: this should be using the CI from the cache, if possible instead: haskey(cache, mi) && (ci = cache[mi])
@@ -1137,7 +1150,7 @@ function typeinf_type(interp::AbstractInterpreter, mi::MethodInstance)
         end
     end
     result = InferenceResult(mi, typeinf_lattice(interp))
-    result.ci = ci
+    result.ci_for_cache = ci
     frame = InferenceState(result, #=cache_mode=#:global, interp)
     if frame === nothing
         engine_reject(interp, ci)