fix compile-all / anticodegen / no-inference modes and remove any type-inferred code that isn't being used

also refactor a bit of compile=all and cache-method code
to make it clearer how jl_is_cacheable_sig parallels it

Author: vtjnash

base/docs/Docs.jl

@@ -59,7 +59,7 @@ include("bindings.jl")
 import Base.Markdown: @doc_str, MD
 import Base.Meta: quot, isexpr
 import Base: Callable
-import Core.Inference.CoreDocs: lazy_iterpolate
+import ..CoreDocs: lazy_iterpolate
 
 export doc
 

base/docs/core.jl

@@ -2,7 +2,7 @@
 
 module CoreDocs
 
-import Core.Inference: esc, push!, getindex, current_module, unsafe_load, Csize_t
+import ..esc, ..push!, ..getindex, ..current_module, ..unsafe_load, ..Csize_t
 
 function doc!(str, ex)
     ptr  = unsafe_load(Core.Intrinsics.cglobal(:jl_filename, Ptr{UInt8}))

base/inference.jl

@@ -71,9 +71,10 @@ type InferenceState
     typegotoredo::Bool
     inworkq::Bool
     optimize::Bool
+    needtree::Bool
     inferred::Bool
 
-    function InferenceState(linfo::LambdaInfo, optimize::Bool)
+    function InferenceState(linfo::LambdaInfo, optimize::Bool, needtree::Bool)
         @assert isa(linfo.code,Array{Any,1})
         nslots = length(linfo.slotnames)
         nl = label_counter(linfo.code)+1
@@ -158,7 +159,7 @@ type InferenceState
             ssavalue_uses, ssavalue_init,
             ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(),
             Vector{Tuple{InferenceState, Vector{LineNum}}}(),
-            false, false, false, optimize, false)
+            false, false, false, optimize, needtree, false)
         push!(active, frame)
         nactive[] += 1
         return frame
@@ -1473,6 +1474,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr
             # don't call staged functions on abstract types.
             # (see issues #8504, #10230)
             # we can't guarantee that their type behavior is monotonic.
+            # XXX: this test is wrong if Types (such as DataType) are present
             return (nothing, Any, false)
         end
         try
@@ -1485,6 +1487,8 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr
         linfo = specialize_method(method, atypes, sparams, cached)
     end
 
+    # XXX: the following logic is likely subtly broken if code.code was nothing,
+    #      although it seems unlikely something bad (infinite recursion) will happen as a result
     if linfo.inInference
         # inference on this signature may be in progress,
         # find the corresponding frame in the active list
@@ -1498,10 +1502,13 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr
         end
         # TODO: this assertion seems iffy
         assert(frame !== nothing)
+        if needtree
+            frame.needtree = true
+        end
     else
         # inference not started yet, make a new frame for a new lambda
         linfo.inInference = true
-        frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize)
+        frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize, needtree)
     end
     frame = frame::InferenceState
 
@@ -1541,8 +1548,8 @@ end
 function typeinf_ext(linfo::LambdaInfo)
     if isdefined(linfo, :def)
         # method lambda - infer this specialization via the method cache
-        (code, _t, _) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo)
-        if code.inferred && linfo !== code
+        (code, _t, inferred) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo)
+        if inferred && code.inferred && linfo !== code
             # This case occurs when the IR for a function has been deleted.
             # `code` will be a newly-created LambdaInfo, and we need to copy its
             # contents to the existing one to copy the info to the method cache.
@@ -1562,7 +1569,7 @@ function typeinf_ext(linfo::LambdaInfo)
     else
         # toplevel lambda - infer directly
         linfo.inInference = true
-        frame = InferenceState(linfo, true)
+        frame = InferenceState(linfo, true, true)
         typeinf_loop(frame)
         @assert frame.inferred # TODO: deal with this better
         return linfo
@@ -1875,7 +1882,8 @@ function isinlineable(linfo::LambdaInfo)
             end
         end
         if !inlineable
-            body = Expr(:block); body.args = linfo.code
+            body = Expr(:block)
+            body.args = linfo.code
             inlineable = inline_worthy(body, cost)
         end
     end
@@ -1903,9 +1911,6 @@ function finish(me::InferenceState)
     end
     type_annotate!(me.linfo, me.stmt_types, me, me.nargs)
 
-    # make sure (meta pure) is stripped from full tree
-    ispure = popmeta!(me.linfo.code, :pure)[1]
-
     # run optimization passes on fulltree
     if me.optimize
         # This pass is required for the AST to be valid in codegen
@@ -1923,22 +1928,34 @@ function finish(me::InferenceState)
         reindex_labels!(me.linfo, me)
     end
     widen_all_consts!(me.linfo)
-
-    # finalize and record the linfo result
-    me.inferred = true
+    # make sure (meta pure) is stripped from full tree
+    ispure = popmeta!(me.linfo.code, :pure)[1]
+    me.linfo.pure = ispure
 
     # determine and cache inlineability
     me.linfo.inlineable = isinlineable(me.linfo)
 
-    if isdefined(me.linfo, :def)
-        # compress code for non-toplevel thunks
-        compressedtree = ccall(:jl_compress_ast, Any, (Any,Any), me.linfo, me.linfo.code)
-        me.linfo.code = compressedtree
+    if !me.needtree
+        me.needtree = me.linfo.inlineable || ccall(:jl_is_cacheable_sig, Int32, (Any, Any, Any),
+            me.linfo.specTypes, me.linfo.def.sig, me.linfo.def) != 0
     end
-    me.linfo.pure = ispure
+
+    if me.needtree
+        if isdefined(me.linfo, :def)
+            # compress code for non-toplevel thunks
+            compressedtree = ccall(:jl_compress_ast, Any, (Any,Any), me.linfo, me.linfo.code)
+            me.linfo.code = compressedtree
+        end
+    else
+        ccall(:jl_set_lambda_code_null, Void, (Any,), me.linfo)
+        me.linfo.inlineable = false
+    end
+
     ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, me.bestguess)
     me.linfo.inferred = true
     me.linfo.inInference = false
+    # finalize and record the linfo result
+    me.inferred = true
 
     # lazy-delete the item from active for several reasons:
     # efficiency, correctness, and recursion-safety

base/sysimg.jl

@@ -12,7 +12,10 @@ eval(m,x) = Core.eval(m,x)
 
 # init core docsystem
 import Core: @doc, @__doc__, @doc_str
-Core.atdoc!(Core.Inference.CoreDocs.docm)
+if isdefined(Core, :Inference)
+    import Core.Inference.CoreDocs
+    Core.atdoc!(CoreDocs.docm)
+end
 
 include("exports.jl")
 
@@ -61,6 +64,12 @@ include("refpointer.jl")
 include("checked.jl")
 importall .Checked
 
+# Symbol constructors
+if !isdefined(Core, :Inference)
+    Symbol(s::String) = Symbol(s.data)
+    Symbol(a::Array{UInt8,1}) =
+        ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int32), a, length(a))
+end
 # vararg Symbol constructor
 Symbol(x...) = Symbol(string(x...))
 
@@ -129,6 +138,11 @@ include("osutils.jl")
 include("c.jl")
 include("sysinfo.jl")
 
+if !isdefined(Core, :Inference)
+    include("docs/core.jl")
+    Core.atdoc!(CoreDocs.docm)
+end
+
 # Core I/O
 include("io.jl")
 include("iostream.jl")
@@ -350,7 +364,7 @@ include("docs/basedocs.jl")
 include("markdown/Markdown.jl")
 include("docs/Docs.jl")
 using .Docs, .Markdown
-Docs.loaddocs(Core.Inference.CoreDocs.DOCS)
+isdefined(Core, :Inference) && Docs.loaddocs(Core.Inference.CoreDocs.DOCS)
 
 function __init__()
     # Base library init

src/alloc.c

@@ -520,18 +520,6 @@ JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t
         new_linfo->specTypes = types;
         new_linfo->def = m;
         new_linfo->sparam_vals = sp;
-
-        if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF ||
-                jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) {
-            // copy fptr from the template method definition
-            new_linfo->fptr = linfo->fptr;
-            new_linfo->jlcall_api = linfo->jlcall_api;
-            if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF && new_linfo->fptr == NULL) {
-                jl_printf(JL_STDERR,"code missing for ");
-                jl_static_show(JL_STDERR, (jl_value_t*)new_linfo);
-                jl_printf(JL_STDERR, "  sysimg may not have been built with --compile=all\n");
-            }
-        }
     }
     else {
         new_linfo = jl_instantiate_staged(m, types, sp);

src/anticodegen.c

@@ -9,44 +9,42 @@ int globalUnique = 0;
 
 #define UNAVAILABLE { jl_errorf("%s: not available in this build of Julia", __func__); }
 
-void jl_dump_bitcode(char *fname, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE
+void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE
 void jl_dump_objfile(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE
 int32_t jl_get_llvm_gv(jl_value_t *p) UNAVAILABLE
 void jl_write_malloc_log(void) UNAVAILABLE
 void jl_write_coverage_data(void) UNAVAILABLE
 void jl_generate_fptr(jl_lambda_info_t *li) UNAVAILABLE
-void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx) UNAVAILABLE
+void jl_compile_linfo(jl_lambda_info_t *li) UNAVAILABLE
 
 JL_DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE
 JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE
 JL_DLLEXPORT void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) UNAVAILABLE
 JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) UNAVAILABLE
 JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, uint8_t strip_ir_metadata, uint8_t dump_module) UNAVAILABLE
 
-JL_DLLEXPORT void *jl_LLVMCreateDisasm(const char *, void *, int, void*, void*) UNAVAILABLE
-JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *, uint64_t, uint64_t PC, char *, size_t) UNAVAILABLE
+JL_DLLEXPORT void *jl_LLVMCreateDisasm(const char *TripleName, void *DisInfo, int TagType, void *GetOpInfo, void *SymbolLookUp) UNAVAILABLE
+JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *Bytes, uint64_t BytesSize, uint64_t PC, char *OutString, size_t OutStringSize) UNAVAILABLE
 
 void jl_init_codegen(void) { }
 void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig)
 {
     if (!specsig)
         lam->fptr = fptr;
 }
-void jl_getFunctionInfo(char **name, char **filename, size_t *line,
-                        char **inlinedat_file, size_t *inlinedat_line, jl_lambda_info_t **outer_linfo,
-                        size_t pointer, int *fromC, int skipC, int skipInline)
+
+int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int noInline)
 {
-    *name = NULL;
-    *line = -1;
-    *filename = NULL;
-    *inlinedat_file = NULL;
-    *inlinedat_line = -1;
-    *outer_linfo = NULL;
-    *fromC = 0;
+    return 0;
 }
 
 jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod,
                            jl_lambda_info_t *li, int sparams, int allow_alloc)
 {
     return NULL;
 }
+
+void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambda_info_t **linfos, size_t n)
+{
+    (void)sysimage_base; (void)fptrs; (void)linfos; (void)n;
+}

src/codegen.cpp

@@ -870,11 +870,8 @@ static void to_function(jl_lambda_info_t *li)
     // if not inlineable, code won't be needed again
     if (JL_DELETE_NON_INLINEABLE &&
         li->def && li->inferred && !li->inlineable &&
-        li != li->def->lambda_template && !jl_options.outputji) {
-        li->code = jl_nothing;
-        li->ssavaluetypes = jl_box_long(jl_array_len(li->ssavaluetypes)); jl_gc_wb(li, li->ssavaluetypes);
-        li->slotflags = NULL;
-        li->slotnames = NULL;
+        li != li->def->lambda_template && !imaging_mode) {
+        jl_set_lambda_code_null(li);
     }
 
     // done compiling: restore global state

src/disasm.cpp

@@ -233,7 +233,7 @@ static int OpInfoLookup(void *DisInfo, uint64_t PC, uint64_t Offset, uint64_t Si
     default: return 0;          // Cannot handle input address size
     }
     int skipC = 0;
-    jl_frame_t *frame;
+    jl_frame_t *frame = NULL;
     jl_getFunctionInfo(&frame, pointer, skipC, 1);
     char *name = frame->func_name;
     free(frame->file_name);

src/dump.c

@@ -1267,6 +1267,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc)
             // builtin types are not serialized, so their caches aren't
             // explicitly saved. so we reconstruct the caches of builtin
             // parametric types here.
+            // XXX: need to make sure to serialize all leaftypes in this cache
+            // that were referenced from compiled code
             jl_array_ptr_1d_push(datatype_list, (jl_value_t*)dt);
         }
     }