From 1246ab3e7e2d63af2de6666f8c6c628391a19991 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 4 Sep 2017 12:28:50 -0400 Subject: [PATCH] fix #17997, don't load packages in `Main` --- base/interactiveutil.jl | 24 ++- base/loading.jl | 104 +++++++++--- base/pkg/entry.jl | 2 +- base/reflection.jl | 31 ++-- base/serialize.jl | 29 ++-- base/show.jl | 4 +- examples/clustermanager/0mq/ZMQCM.jl | 6 +- src/dump.c | 85 +++++----- src/toplevel.c | 229 ++++++++++++++------------- test/arrayops.jl | 2 +- test/compile.jl | 50 +++--- test/examples.jl | 13 -- test/libgit2.jl | 2 +- test/linalg/triangular.jl | 2 +- test/lineedit.jl | 2 +- test/offsetarray.jl | 2 +- test/ranges.jl | 2 +- test/repl.jl | 2 +- test/sets.jl | 2 +- test/statistics.jl | 2 +- 20 files changed, 342 insertions(+), 253 deletions(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 748f075b4b81b..1a59142d8b67e 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -567,7 +567,7 @@ end Return an array of methods with an argument of type `typ`. The optional second argument restricts the search to a particular module or function -(the default is all modules, starting from Main). +(the default is all modules). If optional `showparents` is `true`, also return arguments with a parent type of `typ`, excluding type `Any`. @@ -589,30 +589,29 @@ function methodswith(t::Type, f::Function, showparents::Bool=false, meths = Meth return meths end -function methodswith(t::Type, m::Module, showparents::Bool=false) +function _methodswith(t::Type, m::Module, showparents::Bool, visited = Set{Module}()) + push!(visited, m) meths = Method[] for nm in names(m) if isdefined(m, nm) f = getfield(m, nm) if isa(f, Function) methodswith(t, f, showparents, meths) + elseif isa(f, Module) && !in(f, visited) + append!(meths, _methodswith(t, f::Module, showparents, visited)) end end end return unique(meths) end +methodswith(t::Type, m::Module, showparents::Bool=false) = _methodswith(t, m, showparents) + function methodswith(t::Type, showparents::Bool=false) meths = Method[] - mainmod = Main - # find modules in Main - for nm in names(mainmod) - if isdefined(mainmod, nm) - mod = getfield(mainmod, nm) - if isa(mod, Module) - append!(meths, methodswith(t, mod, showparents)) - end - end + visited = Set{Module}() + for mod in loaded_modules_array() + append!(meths, _methodswith(t, mod, showparents, visited)) end return unique(meths) end @@ -679,8 +678,7 @@ download(url, filename) workspace() Replace the top-level module (`Main`) with a new one, providing a clean workspace. The -previous `Main` module is made available as `LastMain`. A previously-loaded package can be -accessed using a statement such as `using LastMain.Package`. +previous `Main` module is made available as `LastMain`. This function should only be used interactively. """ diff --git a/base/loading.jl b/base/loading.jl index c3be02d771fb7..0a0cb668de336 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -288,6 +288,7 @@ function reload(name::AbstractString) error("use `include` instead of `reload` to load source files") else # reload("Package") is ok + unreference_module(Symbol(name)) require(Symbol(name)) end end @@ -315,25 +316,86 @@ all platforms, including those with case-insensitive filesystems like macOS and Windows. """ function require(mod::Symbol) - _require(mod) + existed = root_module_exists(mod) + M = _require(mod) # After successfully loading, notify downstream consumers if toplevel_load[] && myid() == 1 && nprocs() > 1 # broadcast top-level import/using from node 1 (only) @sync for p in procs() p == 1 && continue @async remotecall_wait(p) do - if !isbindingresolved(Main, mod) || !isdefined(Main, mod) - _require(mod) - end + _require(mod) + nothing end end end - for callback in package_callbacks - invokelatest(callback, mod) + if !existed + for callback in package_callbacks + invokelatest(callback, mod) + end + end + return M +end + +const loaded_modules = ObjectIdDict() +const module_keys = ObjectIdDict() + +function register_root_module(key, m::Module) + if haskey(loaded_modules, key) + oldm = loaded_modules[key] + if oldm !== m + name = module_name(oldm) + warn("replacing module $name.") + end + end + loaded_modules[key] = m + module_keys[m] = key + nothing +end + +register_root_module(:Core, Core) +register_root_module(:Base, Base) +register_root_module(:Main, Main) + +is_root_module(m::Module) = haskey(module_keys, m) + +root_module_key(m::Module) = module_keys[m] + +# This is used as the current module when loading top-level modules. +# It has the special behavior that modules evaluated in it get added +# to the loaded_modules table instead of getting bindings. +baremodule __toplevel__ +using Base +end + +# get a top-level Module from the given key +# for now keys can only be Symbols, but that will change +root_module(key::Symbol) = loaded_modules[key] + +root_module_exists(key::Symbol) = haskey(loaded_modules, key) + +loaded_modules_array() = collect(values(loaded_modules)) + +function unreference_module(key) + if haskey(loaded_modules, key) + m = pop!(loaded_modules, key) + # need to ensure all modules are GC rooted; will still be referenced + # in module_keys + end +end + +function register_all(a) + for m in a + if module_parent(m) === m + register_root_module(module_name(m), m) + end end end function _require(mod::Symbol) + if root_module_exists(mod) + return root_module(mod) + end # dependency-tracking is only used for one top-level include(path), # and is not applied recursively to imported modules: old_track_dependencies = _track_dependencies[] @@ -345,7 +407,7 @@ function _require(mod::Symbol) if loading !== false # load already in progress for this module wait(loading) - return + return root_module(mod) end package_locks[mod] = Condition() @@ -364,7 +426,8 @@ function _require(mod::Symbol) if JLOptions().use_compilecache != 0 doneprecompile = _require_search_from_serialized(mod, path) if !isa(doneprecompile, Bool) - return # success + register_all(doneprecompile) + return root_module(mod) # success end end @@ -391,14 +454,17 @@ function _require(mod::Symbol) warn(m, prefix="WARNING: ") # fall-through, TODO: disable __precompile__(true) error so that the normal include will succeed else - return # success + register_all(m) + return root_module(mod) # success end end # just load the file normally via include # for unknown dependencies + local M try - Base.include_relative(Main, path) + Base.include_relative(__toplevel__, path) + return root_module(mod) catch ex if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true) rethrow() # rethrow non-precompilable=true errors @@ -411,6 +477,8 @@ function _require(mod::Symbol) # TODO: disable __precompile__(true) error and do normal include instead of error error("Module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end + register_all(m) + return root_module(mod) end finally toplevel_load[] = last @@ -532,7 +600,7 @@ function create_expr_cache(input::String, output::String, concrete_deps::Vector{ task_local_storage()[:SOURCE_PATH] = $(source) end) end - serialize(in, :(Base.include(Main, $(abspath(input))))) + serialize(in, :(Base.include(Base.__toplevel__, $(abspath(input))))) if source !== nothing serialize(in, :(delete!(task_local_storage(), :SOURCE_PATH))) end @@ -570,15 +638,9 @@ function compilecache(name::String) cachefile::String = abspath(cachepath, name*".ji") # build up the list of modules that we want the precompile process to preserve concrete_deps = copy(_concrete_dependencies) - for existing in names(Main) - if isdefined(Main, existing) - mod = getfield(Main, existing) - if isa(mod, Module) && !(mod === Main || mod === Core || mod === Base) - mod = mod::Module - if module_parent(mod) === Main && module_name(mod) === existing - push!(concrete_deps, (existing, module_uuid(mod))) - end - end + for (key,mod) in loaded_modules + if !(mod === Main || mod === Core || mod === Base) + push!(concrete_deps, (key, module_uuid(mod))) end end # run the expression and cache the result @@ -675,7 +737,7 @@ function stale_cachefile(modpath::String, cachefile::String) if mod == :Main || mod == :Core || mod == :Base continue # Module is already loaded - elseif isbindingresolved(Main, mod) + elseif root_module_exists(mod) continue end name = string(mod) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 6dc47473cd17b..6bd3a0f2193ba 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -541,7 +541,7 @@ function resolve( info("$(up)grading $pkg: v$ver1 => v$ver2") Write.update(pkg, Read.sha1(pkg,ver2)) pkgsym = Symbol(pkg) - if Base.isbindingresolved(Main, pkgsym) && isa(getfield(Main, pkgsym), Module) + if Base.root_module_exists(pkgsym) push!(imported, "- $pkg") end end diff --git a/base/reflection.jl b/base/reflection.jl index be5f2fc38cb95..186fa73e52a6e 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -56,13 +56,17 @@ julia> fullname(Main) ``` """ function fullname(m::Module) - m === Main && return () - m === Base && return (:Base,) # issue #10653 mn = module_name(m) + if m === Main || m === Base || m === Core + return (mn,) + end mp = module_parent(m) if mp === m - # not Main, but is its own parent, means a prior Main module - n = () + if mn !== :Main + return (mn,) + end + # top-level module, not Main, called :Main => prior Main module + n = (:Main,) this = Main while this !== m if isdefined(this, :LastMain) @@ -516,15 +520,22 @@ function _subtypes(m::Module, x::Union{DataType,UnionAll}, end return sts end -function subtypes(m::Module, x::Union{DataType,UnionAll}) - if isabstract(x) - sort!(collect(_subtypes(m, x)), by=string) - else + +function _subtypes_in(mods::Array, x::Union{DataType,UnionAll}) + if !isabstract(x) # Fast path - Union{DataType,UnionAll}[] + return Union{DataType,UnionAll}[] + end + sts = Set{Union{DataType,UnionAll}}() + visited = Set{Module}() + for m in mods + _subtypes(m, x, sts, visited) end + return sort!(collect(sts), by=string) end +subtypes(m::Module, x::Union{DataType,UnionAll}) = _subtypes_in([m], x) + """ subtypes(T::DataType) @@ -541,7 +552,7 @@ julia> subtypes(Integer) Unsigned ``` """ -subtypes(x::Union{DataType,UnionAll}) = subtypes(Main, x) +subtypes(x::Union{DataType,UnionAll}) = _subtypes_in(loaded_modules_array(), x) function to_tuple_type(@nospecialize(t)) @_pure_meta diff --git a/base/serialize.jl b/base/serialize.jl index b78b658430fe5..6d3c9c0f8c4f0 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -343,9 +343,10 @@ function serialize(s::AbstractSerializer, d::Dict) end function serialize_mod_names(s::AbstractSerializer, m::Module) - p = module_parent(m) - if m !== p - serialize_mod_names(s, p) + if Base.is_root_module(m) + serialize(s, Base.root_module_key(m)) + else + serialize_mod_names(s, module_parent(m)) serialize(s, module_name(m)) end end @@ -772,21 +773,25 @@ function deserialize_svec(s::AbstractSerializer) end function deserialize_module(s::AbstractSerializer) - path = deserialize(s) - m = Main - if isa(path,Tuple) && path !== () - # old version - for mname in path - m = getfield(m,mname)::Module + mkey = deserialize(s) + if isa(mkey, Tuple) + # old version, TODO: remove + if mkey === () + return Main + end + m = Base.root_module(mkey[1]) + for i = 2:length(mkey) + m = getfield(m, mkey[i])::Module end else - mname = path + m = Base.root_module(mkey) + mname = deserialize(s) while mname !== () - m = getfield(m,mname)::Module + m = getfield(m, mname)::Module mname = deserialize(s) end end - m + return m end function deserialize(s::AbstractSerializer, ::Type{Method}) diff --git a/base/show.jl b/base/show.jl index ab517f9c44795..ef77cf5aa8f51 100644 --- a/base/show.jl +++ b/base/show.jl @@ -380,8 +380,8 @@ function show(io::IO, p::Pair) end function show(io::IO, m::Module) - if m === Main - print(io, "Main") + if is_root_module(m) + print(io, module_name(m)) else print(io, join(fullname(m),".")) end diff --git a/examples/clustermanager/0mq/ZMQCM.jl b/examples/clustermanager/0mq/ZMQCM.jl index d18e7b66d58d0..658c5b9f438d4 100644 --- a/examples/clustermanager/0mq/ZMQCM.jl +++ b/examples/clustermanager/0mq/ZMQCM.jl @@ -1,6 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using ZMQ +# the 0mq clustermanager depends on package ZMQ. For testing purposes, at least +# make sure the code loads without it. +try + using ZMQ +end import Base: launch, manage, connect, kill diff --git a/src/dump.c b/src/dump.c index ba2df8034321f..1f2df9db07e37 100644 --- a/src/dump.c +++ b/src/dump.c @@ -370,14 +370,17 @@ static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m) writetag(s->s, jl_module_type); jl_serialize_value(s, m->name); int ref_only = 0; - if (!module_in_worklist(m)) - ref_only = 1; + if (!module_in_worklist(m)) { + if (m == m->parent) + ref_only = 2; + else + ref_only = 1; + } write_int8(s->s, ref_only); - jl_serialize_value(s, m->parent); - if (ref_only) { - assert(m->parent != m); + if (ref_only != 2) + jl_serialize_value(s, m->parent); + if (ref_only) return; - } size_t i; void **table = m->bindings.table; for(i=1; i < m->bindings.size; i+=2) { @@ -994,28 +997,19 @@ static void jl_collect_backedges(jl_array_t *s) } } -// serialize information about all of the modules accessible directly from Main -static void write_mod_list(ios_t *s) +// serialize information about all loaded modules +static void write_mod_list(ios_t *s, jl_array_t *a) { - jl_module_t *m = jl_main_module; size_t i; - void **table = m->bindings.table; - for (i = 1; i < m->bindings.size; i += 2) { - if (table[i] != HT_NOTFOUND) { - jl_binding_t *b = (jl_binding_t*)table[i]; - if (b->owner == m && - b->value && b->constp && - jl_is_module(b->value) && - !module_in_worklist((jl_module_t*)b->value)) { - jl_module_t *child = (jl_module_t*)b->value; - if (child->name == b->name) { - // this is the original/primary binding for the submodule - size_t l = strlen(jl_symbol_name(child->name)); - write_int32(s, l); - ios_write(s, jl_symbol_name(child->name), l); - write_uint64(s, child->uuid); - } - } + size_t len = jl_array_len(a); + for (i = 0; i < len; i++) { + jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(a, i); + assert(jl_is_module(m)); + if (!module_in_worklist(m)) { + size_t l = strlen(jl_symbol_name(m->name)); + write_int32(s, l); + ios_write(s, jl_symbol_name(m->name), l); + write_uint64(s, m->uuid); } } write_int32(s, 0); @@ -1045,7 +1039,7 @@ static void write_work_list(ios_t *s) int i, l = jl_array_len(serializer_worklist); for (i = 0; i < l; i++) { jl_module_t *workmod = (jl_module_t*)jl_array_ptr_ref(serializer_worklist, i); - if (workmod->parent == jl_main_module) { + if (workmod->parent == jl_main_module || workmod->parent == workmod) { size_t l = strlen(jl_symbol_name(workmod->name)); write_int32(s, l); ios_write(s, jl_symbol_name(workmod->name), l); @@ -1505,6 +1499,8 @@ static jl_value_t *jl_deserialize_value_method_instance(jl_serializer_state *s, return (jl_value_t*)li; } +jl_module_t *jl_lookup_root_module(jl_value_t *key); + static jl_value_t *jl_deserialize_value_module(jl_serializer_state *s) { int usetable = (s->mode != MODE_AST); @@ -1514,7 +1510,11 @@ static jl_value_t *jl_deserialize_value_module(jl_serializer_state *s) jl_sym_t *mname = (jl_sym_t*)jl_deserialize_value(s, NULL); int ref_only = read_uint8(s->s); if (ref_only) { - jl_value_t *m_ref = jl_get_global((jl_module_t*)jl_deserialize_value(s, NULL), mname); + jl_value_t *m_ref; + if (ref_only == 1) + m_ref = jl_get_global((jl_module_t*)jl_deserialize_value(s, NULL), mname); + else + m_ref = (jl_value_t*)jl_lookup_root_module((jl_value_t*)mname); if (usetable) backref_list.items[pos] = m_ref; return m_ref; @@ -1910,21 +1910,18 @@ static jl_value_t *read_verify_mod_list(ios_t *s, arraylist_t *dependent_worlds) uint64_t uuid = read_uint64(s); jl_sym_t *sym = jl_symbol(name); jl_module_t *m = NULL; - if (jl_binding_resolved_p(jl_main_module, sym)) - m = (jl_module_t*)jl_get_global(jl_main_module, sym); if (!m) { static jl_value_t *require_func = NULL; if (!require_func) require_func = jl_get_global(jl_base_module, jl_symbol("require")); jl_value_t *reqargs[2] = {require_func, (jl_value_t*)sym}; JL_TRY { - jl_apply(reqargs, 2); + m = (jl_module_t*)jl_apply(reqargs, 2); } JL_CATCH { ios_close(s); jl_rethrow(); } - m = (jl_module_t*)jl_get_global(jl_main_module, sym); } if (!m) { return jl_get_exceptionf(jl_errorexception_type, @@ -2003,6 +2000,8 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) } case 2: { // reinsert module v into parent (const) jl_module_t *mod = (jl_module_t*)v; + if (mod->parent == mod) // top level modules handled by loader + break; jl_binding_t *b = jl_get_binding_wr(mod->parent, mod->name, 1); jl_declare_constant(b); // this can throw if (b->value != NULL) { @@ -2013,8 +2012,7 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) if (jl_generating_output() && jl_options.incremental) { jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name)); } - jl_printf(JL_STDERR, "WARNING: replacing module %s.\n", - jl_symbol_name(mod->name)); + jl_printf(JL_STDERR, "WARNING: replacing module %s.\n", jl_symbol_name(mod->name)); } b->value = v; jl_gc_wb_binding(b, v); @@ -2262,16 +2260,20 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) { char *tmpfname = strcat(strcpy((char *) alloca(strlen(fname)+8), fname), ".XXXXXX"); ios_t f; + jl_array_t *mod_array; if (ios_mkstemp(&f, tmpfname) == NULL) { jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", tmpfname); return 1; } + JL_GC_PUSH1(&mod_array); + mod_array = (jl_array_t*)jl_call0((jl_function_t*)jl_get_global(jl_base_module, jl_symbol("loaded_modules_array"))); + serializer_worklist = worklist; write_header(&f); write_work_list(&f); write_dependency_list(&f); - write_mod_list(&f); // this can return errors during deserialize, - // best to keep it early (before any actual initialization) + write_mod_list(&f, mod_array); // this can return errors during deserialize, + // best to keep it early (before any actual initialization) arraylist_new(&reinit_list, 0); htable_new(&edges_map, 0); @@ -2283,7 +2285,16 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) int en = jl_gc_enable(0); // edges map is not gc-safe jl_array_t *lambdas = jl_alloc_vec_any(0); jl_array_t *edges = jl_alloc_vec_any(0); - jl_collect_lambdas_from_mod(lambdas, jl_main_module); + + size_t i; + size_t len = jl_array_len(mod_array); + for (i = 0; i < len; i++) { + jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); + assert(jl_is_module(m)); + jl_collect_lambdas_from_mod(lambdas, m); + } + JL_GC_POP(); + jl_collect_backedges(edges); jl_serializer_state s = { diff --git a/src/toplevel.c b/src/toplevel.c index 9a56455b7aed8..13573a6d97587 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -120,6 +120,25 @@ static void jl_module_load_time_initialize(jl_module_t *m) } } +jl_module_t *jl_lookup_root_module(jl_value_t *key) +{ + jl_value_t *loaded_modules = jl_get_global(jl_base_module, jl_symbol("loaded_modules")); + jl_array_t *ht = (jl_array_t*)jl_get_nth_field(loaded_modules, 0); + assert(jl_is_array(ht)); + return (jl_module_t*)jl_eqtable_get(ht, key, NULL); +} + +void jl_register_root_module(jl_value_t *key, jl_module_t *m) +{ + static jl_value_t *register_module_func=NULL; + if (register_module_func == NULL && jl_base_module != NULL) + register_module_func = jl_get_global(jl_base_module, jl_symbol("register_root_module")); + if (register_module_func != NULL) { + jl_value_t *rmargs[3] = {register_module_func, key, (jl_value_t*)m}; + jl_apply(rmargs, 3); + } +} + jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -140,32 +159,34 @@ jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex) if (!jl_is_symbol(name)) { jl_type_error("module", (jl_value_t*)jl_sym_type, (jl_value_t*)name); } - jl_binding_t *b = jl_get_binding_wr(parent_module, name, 1); - jl_declare_constant(b); - if (b->value != NULL) { - if (!jl_is_module(b->value)) { - jl_errorf("invalid redefinition of constant %s", - jl_symbol_name(name)); - } - if (jl_generating_output()) { - jl_errorf("cannot replace module %s during compilation", - jl_symbol_name(name)); + jl_module_t *newm = jl_new_module(name); + if (jl_base_module && + (jl_value_t*)parent_module == jl_get_global(jl_base_module, jl_symbol("__toplevel__"))) { + newm->parent = newm; + // TODO: pass through correct key somehow + jl_register_root_module((jl_value_t*)name, newm); + } + else { + jl_binding_t *b = jl_get_binding_wr(parent_module, name, 1); + jl_declare_constant(b); + if (b->value != NULL) { + if (!jl_is_module(b->value)) { + jl_errorf("invalid redefinition of constant %s", jl_symbol_name(name)); + } + if (jl_generating_output()) { + jl_errorf("cannot replace module %s during compilation", jl_symbol_name(name)); + } + jl_printf(JL_STDERR, "WARNING: replacing module %s.\n", jl_symbol_name(name)); } - jl_printf(JL_STDERR, "WARNING: replacing module %s\n", - jl_symbol_name(name)); + newm->parent = parent_module; + b->value = (jl_value_t*)newm; + jl_gc_wb_binding(b, newm); } - jl_module_t *newm = jl_new_module(name); - newm->parent = parent_module; - b->value = (jl_value_t*)newm; - jl_gc_wb_binding(b, newm); if (parent_module == jl_main_module && name == jl_symbol("Base")) { // pick up Base module during bootstrap jl_base_module = newm; } - // export all modules from Main - if (parent_module == jl_main_module) - jl_module_export(jl_main_module, name); // add standard imports unless baremodule if (std_imports) { @@ -339,99 +360,70 @@ static int jl_eval_expr_with_compiler_p(jl_value_t *e, int compileloops, jl_modu return 0; } -static jl_value_t *require_func=NULL; - -static jl_module_t *eval_import_path_(jl_module_t *from, jl_array_t *args, int retrying) +// either: +// - sets *name and returns the module to import *name from +// - sets *name to NULL and returns a module to import +static jl_module_t *eval_import_path(jl_module_t *from, jl_array_t *args, jl_sym_t **name, const char *keyword) { - // in .A.B.C, first find a binding for A in the chain of module scopes - // following parent links. then evaluate the rest of the path from there. - // in A.B, look for A in Main first. + static jl_value_t *require_func=NULL; jl_sym_t *var = (jl_sym_t*)jl_array_ptr_ref(args, 0); size_t i = 1; + jl_module_t *m = NULL; + *name = NULL; if (!jl_is_symbol(var)) - jl_type_error("import or using", (jl_value_t*)jl_sym_type, (jl_value_t*)var); + jl_type_error(keyword, (jl_value_t*)jl_sym_type, (jl_value_t*)var); - jl_module_t *m; if (var != dot_sym) { - m = jl_main_module; + // `A.B`: call the loader to obtain the root A in the current environment. + if (jl_core_module && var == jl_core_module->name) { + m = jl_core_module; + } + else if (jl_base_module && var == jl_base_module->name) { + m = jl_base_module; + } + else { + if (require_func == NULL && jl_base_module != NULL) + require_func = jl_get_global(jl_base_module, jl_symbol("require")); + if (require_func != NULL) { + jl_value_t *reqargs[2] = {require_func, (jl_value_t*)var}; + m = (jl_module_t*)jl_apply(reqargs, 2); + } + if (m == NULL || !jl_is_module(m)) { + jl_errorf("failed to load module %s", jl_symbol_name(var)); + } + } + if (i == jl_array_len(args)) + return m; } else { + // `.A.B.C`: strip off leading dots by following parent links m = from; while (1) { if (i >= jl_array_len(args)) jl_error("invalid module path"); var = (jl_sym_t*)jl_array_ptr_ref(args, i); - if (!jl_is_symbol(var)) - jl_type_error("import or using", (jl_value_t*)jl_sym_type, (jl_value_t*)var); + if (var != dot_sym) + break; i++; - if (var != dot_sym) { - if (i == jl_array_len(args)) - return m; - else - break; - } m = m->parent; } } while (1) { - if (jl_binding_resolved_p(m, var)) { - jl_binding_t *mb = jl_get_binding(m, var); - jl_module_t *m0 = m; - int isimp = jl_is_imported(m, var); - assert(mb != NULL); - if (mb->owner == m0 || isimp) { - m = (jl_module_t*)mb->value; - if ((mb->owner == m0 && m != NULL && !jl_is_module(m)) || - (isimp && (m == NULL || !jl_is_module(m)))) - jl_errorf("invalid module path (%s does not name a module)", - jl_symbol_name(var)); - // If the binding has been resolved but is (1) undefined, and (2) owned - // by the module we're importing into, then allow the import into the - // undefined variable (by setting m back to m0). - if (m == NULL) - m = m0; - else - break; - } - } - if (m == jl_main_module) { - if (!retrying && i==1) { // (i==1) => no require() for relative imports - if (require_func == NULL && jl_base_module != NULL) - require_func = jl_get_global(jl_base_module, jl_symbol("require")); - if (require_func != NULL) { - jl_value_t *reqargs[2] = {require_func, (jl_value_t*)var}; - jl_apply(reqargs, 2); - return eval_import_path_(from, args, 1); - } - } - } - if (retrying && require_func) { - jl_printf(JL_STDERR, "WARNING: requiring \"%s\" in module \"%s\" did not define a corresponding module.\n", - jl_symbol_name(var), - jl_symbol_name(from->name)); - return NULL; - } - else { - jl_errorf("in module path: %s not defined", jl_symbol_name(var)); - } - } - - for(; i < jl_array_len(args)-1; i++) { - jl_value_t *s = jl_array_ptr_ref(args,i); - assert(jl_is_symbol(s)); - m = (jl_module_t*)jl_eval_global_var(m, (jl_sym_t*)s); + var = (jl_sym_t*)jl_array_ptr_ref(args, i); + if (!jl_is_symbol(var)) + jl_type_error(keyword, (jl_value_t*)jl_sym_type, (jl_value_t*)var); + if (i == jl_array_len(args)-1) + break; + m = (jl_module_t*)jl_eval_global_var(m, var); if (!jl_is_module(m)) - jl_errorf("invalid import statement"); + jl_errorf("invalid %s path: \"%s\" does not name a module", keyword, jl_symbol_name(var)); + i++; } + *name = var; return m; } -static jl_module_t *eval_import_path(jl_module_t *from, jl_array_t *args) -{ - return eval_import_path_(from, args, 0); -} - jl_value_t *jl_toplevel_eval_body(jl_module_t *m, jl_array_t *stmts); int jl_is_toplevel_only_expr(jl_value_t *e) @@ -464,6 +456,19 @@ static jl_method_instance_t *jl_new_thunk(jl_code_info_t *src, jl_module_t *modu return li; } +static void import_module(jl_module_t *m, jl_module_t *import) +{ + jl_sym_t *name = import->name; + if (jl_binding_resolved_p(m, name)) { + jl_binding_t *b = jl_get_binding(m, name); + if (b->owner != m || (b->value && b->value != (jl_value_t*)import)) { + jl_errorf("importing %s into %s conflicts with an existing identifier", + jl_symbol_name(name), jl_symbol_name(m->name)); + } + } + jl_set_const(m, name, (jl_value_t*)import); +} + jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -484,28 +489,29 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int e return jl_eval_module_expr(m, ex); } else if (ex->head == importall_sym) { - jl_module_t *import = eval_import_path(m, ex->args); - if (import == NULL) - return jl_nothing; - jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, jl_array_len(ex->args) - 1); - if (!jl_is_symbol(name)) - jl_error("syntax: malformed \"importall\" statement"); - import = (jl_module_t*)jl_eval_global_var(import, name); - if (!jl_is_module(import)) - jl_errorf("invalid %s statement: name exists but does not refer to a module", jl_symbol_name(ex->head)); + jl_sym_t *name = NULL; + jl_module_t *import = eval_import_path(m, ex->args, &name, "importall"); + if (name != NULL) { + import = (jl_module_t*)jl_eval_global_var(import, name); + if (!jl_is_module(import)) + jl_errorf("invalid %s statement: name exists but does not refer to a module", jl_symbol_name(ex->head)); + } jl_module_importall(m, import); return jl_nothing; } else if (ex->head == using_sym) { - jl_module_t *import = eval_import_path(m, ex->args); - if (import == NULL) - return jl_nothing; - jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, jl_array_len(ex->args) - 1); - if (!jl_is_symbol(name)) - jl_error("syntax: malformed \"using\" statement"); - jl_module_t *u = (jl_module_t*)jl_eval_global_var(import, name); + jl_sym_t *name = NULL; + jl_module_t *import = eval_import_path(m, ex->args, &name, "using"); + jl_module_t *u = import; + if (name != NULL) + u = (jl_module_t*)jl_eval_global_var(import, name); if (jl_is_module(u)) { jl_module_using(m, u); + if (m == jl_main_module && name == NULL) { + // TODO: for now, `using A` in Main also creates an explicit binding for `A` + // This will possibly be extended to all modules. + import_module(m, u); + } } else { jl_module_use(m, import, name); @@ -513,13 +519,14 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int e return jl_nothing; } else if (ex->head == import_sym) { - jl_module_t *import = eval_import_path(m, ex->args); - if (import == NULL) - return jl_nothing; - jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, jl_array_len(ex->args) - 1); - if (!jl_is_symbol(name)) - jl_error("syntax: malformed \"import\" statement"); - jl_module_import(m, import, name); + jl_sym_t *name = NULL; + jl_module_t *import = eval_import_path(m, ex->args, &name, "import"); + if (name == NULL) { + import_module(m, import); + } + else { + jl_module_import(m, import, name); + } return jl_nothing; } else if (ex->head == export_sym) { diff --git a/test/arrayops.jl b/test/arrayops.jl index 0fe17e2561213..34101b99d9019 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -2,7 +2,7 @@ # Array test isdefined(Main, :TestHelpers) || @eval Main include("TestHelpers.jl") -using TestHelpers.OAs +using Main.TestHelpers.OAs @testset "basics" begin @test length([1, 2, 3]) == 3 diff --git a/test/compile.jl b/test/compile.jl index e47f1128de02a..758396f7e2676 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -2,6 +2,8 @@ using Base.Test +import Base: root_module + Foo_module = :Foo4b3a94a1a081a8cb Foo2_module = :F2oo4b3a94a1a081a8cb FooBase_module = :FooBase4b3a94a1a081a8cb @@ -149,7 +151,7 @@ try # Issue #21307 Base.require(Foo2_module) @eval let Foo2_module = $(QuoteNode(Foo2_module)), # use @eval to see the results of loading the compile - Foo = getfield(Main, Foo2_module) + Foo = root_module(Foo2_module) Foo.override(::Int) = 'a' Foo.override(::Float32) = 'b' end @@ -157,7 +159,7 @@ try Base.require(Foo_module) @eval let Foo_module = $(QuoteNode(Foo_module)), # use @eval to see the results of loading the compile - Foo = getfield(Main, Foo_module) + Foo = root_module(Foo_module) @test Foo.foo(17) == 18 @test Foo.Bar.bar(17) == 19 @@ -172,16 +174,18 @@ try # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: @test_warn "WARNING: replacing module $Foo_module." begin - @test isa(Base._require_from_serialized(Foo_module, cachefile), Array{Any,1}) + ms = Base._require_from_serialized(Foo_module, cachefile) + @test isa(ms, Array{Any,1}) + Base.register_all(ms) end - let Foo = getfield(Main, Foo_module) + let Foo = root_module(Foo_module) @test_throws MethodError Foo.foo(17) # world shouldn't be visible yet end @eval let Foo_module = $(QuoteNode(Foo_module)), # use @eval to see the results of loading the compile Foo2_module = $(QuoteNode(Foo2_module)), FooBase_module = $(QuoteNode(FooBase_module)), - Foo = getfield(Main, Foo_module), + Foo = root_module(Foo_module), dir = $(QuoteNode(dir)), cachefile = $(QuoteNode(cachefile)), Foo_file = $(QuoteNode(Foo_file)) @@ -291,7 +295,7 @@ try @test !Base.stale_cachefile(relFooBar_file, joinpath(dir, "FooBar.ji")) @eval using FooBar - fb_uuid = Base.module_uuid(Main.FooBar) + fb_uuid = Base.module_uuid(FooBar) sleep(2); touch(FooBar_file) insert!(Base.LOAD_CACHE_PATH, 1, dir2) @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) @@ -301,22 +305,22 @@ try @test isfile(joinpath(dir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) - @test fb_uuid == Base.module_uuid(Main.FooBar) - fb_uuid1 = Base.module_uuid(Main.FooBar1) + @test fb_uuid == Base.module_uuid(FooBar) + fb_uuid1 = Base.module_uuid(FooBar1) @test fb_uuid != fb_uuid1 - @test_warn "WARNING: replacing module FooBar." reload("FooBar") - @test fb_uuid != Base.module_uuid(Main.FooBar) - @test fb_uuid1 == Base.module_uuid(Main.FooBar1) - fb_uuid = Base.module_uuid(Main.FooBar) + reload("FooBar") + @test fb_uuid != Base.module_uuid(root_module(:FooBar)) + @test fb_uuid1 == Base.module_uuid(FooBar1) + fb_uuid = Base.module_uuid(root_module(:FooBar)) @test isfile(joinpath(dir2, "FooBar.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji")) - @test_warn "WARNING: replacing module FooBar1." reload("FooBar1") - @test fb_uuid == Base.module_uuid(Main.FooBar) - @test fb_uuid1 != Base.module_uuid(Main.FooBar1) + reload("FooBar1") + @test fb_uuid == Base.module_uuid(root_module(:FooBar)) + @test fb_uuid1 != Base.module_uuid(root_module(:FooBar1)) @test isfile(joinpath(dir2, "FooBar.ji")) @test isfile(joinpath(dir2, "FooBar1.ji")) @@ -331,15 +335,16 @@ try @test Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) # test behavior of precompile modules that throw errors - write(FooBar_file, + FooBar2_file = joinpath(dir, "FooBar2.jl") + write(FooBar2_file, """ __precompile__(true) - module FooBar + module FooBar2 error("break me") end """) @test_warn "ERROR: LoadError: break me\nStacktrace:\n [1] error" try - Base.require(:FooBar) + Base.require(:FooBar2) error("\"LoadError: break me\" test failed") catch exc isa(exc, ErrorException) || rethrow(exc) @@ -379,7 +384,7 @@ try """) rm(FooBarT_file) @test Base.stale_cachefile(FooBarT2_file, joinpath(dir2, "FooBarT2.ji")) - @test Base.require(:FooBarT2) === nothing + @test Base.require(:FooBarT2) isa Module finally splice!(Base.LOAD_CACHE_PATH, 1:2) splice!(LOAD_PATH, 1) @@ -513,7 +518,6 @@ let dir = mktempdir() Base.compilecache("$(Test2_module)") @test !Base.isbindingresolved(Main, Test2_module) Base.require(Test2_module) - @test Base.isbindingresolved(Main, Test2_module) @test take!(loaded_modules) == Test1_module @test take!(loaded_modules) == Test2_module write(joinpath(dir, "$(Test3_module).jl"), @@ -541,7 +545,7 @@ let module_name = string("a",randstring()) code = """module $(module_name)\nend\n""" write(file_name, code) reload(module_name) - @test isa(getfield(Main, Symbol(module_name)), Module) + @test isa(root_module(Symbol(module_name)), Module) @test shift!(LOAD_PATH) == path rm(file_name) end @@ -586,9 +590,9 @@ let end try @eval using $ModuleB - uuid = Base.module_uuid(getfield(Main, ModuleB)) + uuid = Base.module_uuid(root_module(ModuleB)) for wid in test_workers - @test Base.Distributed.remotecall_eval(Main, wid, :( Base.module_uuid($ModuleB) )) == uuid + @test Base.Distributed.remotecall_eval(Main, wid, :( Base.module_uuid(Base.root_module($(QuoteNode(ModuleB)))) )) == uuid if wid != myid() # avoid world-age errors on the local proc @test remotecall_fetch(g, wid) == wid end diff --git a/test/examples.jl b/test/examples.jl index c406c5784ca5e..ebc6cdcdd0531 100644 --- a/test/examples.jl +++ b/test/examples.jl @@ -75,17 +75,4 @@ put!(dc, "Hello", "World") # At least make sure code loads include(joinpath(dir, "wordcount.jl")) -# the 0mq clustermanager depends on package ZMQ. Just making sure the -# code loads using a stub module definition for ZMQ. -zmq_found=true -try - using ZMQ -catch - zmq_found=false -end - -if !zmq_found - eval(Main, parse("module ZMQ end")) -end - include(joinpath(dir, "clustermanager/0mq/ZMQCM.jl")) diff --git a/test/libgit2.jl b/test/libgit2.jl index 72c964cb9be0a..459f197d3d073 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license isdefined(Main, :TestHelpers) || @eval Main include(joinpath(@__DIR__, "TestHelpers.jl")) -import TestHelpers: challenge_prompt +import Main.TestHelpers: challenge_prompt const LIBGIT2_MIN_VER = v"0.23.0" const LIBGIT2_HELPER_PATH = joinpath(@__DIR__, "libgit2-helpers.jl") diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index ff03cbcd2f26c..f4db30dc7e087 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -509,7 +509,7 @@ end # dimensional correctness: isdefined(Main, :TestHelpers) || @eval Main include("../TestHelpers.jl") -using TestHelpers.Furlong +using Main.TestHelpers.Furlong let A = UpperTriangular([Furlong(1) Furlong(4); Furlong(0) Furlong(1)]) @test sqrt(A) == Furlong{1//2}.(UpperTriangular([1 2; 0 1])) end diff --git a/test/lineedit.jl b/test/lineedit.jl index cb275d37d4da1..f0600e3314396 100644 --- a/test/lineedit.jl +++ b/test/lineedit.jl @@ -4,7 +4,7 @@ using Base.LineEdit using Base.LineEdit: edit_insert, buffer, content, setmark, getmark isdefined(Main, :TestHelpers) || @eval Main include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) -using TestHelpers +using Main.TestHelpers # no need to have animation in tests LineEdit.REGION_ANIMATION_DURATION[] = 0.001 diff --git a/test/offsetarray.jl b/test/offsetarray.jl index b2c642cdfced3..fde131b7f2e12 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license isdefined(Main, :TestHelpers) || @eval Main include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) -using TestHelpers.OAs +using Main.TestHelpers.OAs const OAs_name = join(fullname(OAs), ".") diff --git a/test/ranges.jl b/test/ranges.jl index a318f04b2c0f0..6eefc33123aa5 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -1133,7 +1133,7 @@ Base.isless(x, y::NotReal) = isless(x, y.val) # dimensional correctness: isdefined(Main, :TestHelpers) || @eval Main include("TestHelpers.jl") -using TestHelpers.Furlong +using Main.TestHelpers.Furlong @test_throws MethodError collect(Furlong(2):Furlong(10)) # step size is ambiguous @test_throws MethodError range(Furlong(2), 9) # step size is ambiguous @test collect(Furlong(2):Furlong(1):Furlong(10)) == collect(range(Furlong(2),Furlong(1),9)) == Furlong.(2:10) diff --git a/test/repl.jl b/test/repl.jl index 31bd210b7f58b..684faa1eefdb0 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -5,7 +5,7 @@ include("testenv.jl") # REPL tests isdefined(Main, :TestHelpers) || @eval Main include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) -using TestHelpers +using Main.TestHelpers import Base: REPL, LineEdit function fake_repl(f) diff --git a/test/sets.jl b/test/sets.jl index 7041e475295b1..5a7da84d4a656 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -2,7 +2,7 @@ # Set tests isdefined(Main, :TestHelpers) || @eval Main include("TestHelpers.jl") -using TestHelpers.OAs +using Main.TestHelpers.OAs # Construction, collect @test ===(typeof(Set([1,2,3])), Set{Int}) diff --git a/test/statistics.jl b/test/statistics.jl index 2962853b42eb9..638e8d670378e 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -405,7 +405,7 @@ end # dimensional correctness isdefined(Main, :TestHelpers) || @eval Main include("TestHelpers.jl") -using TestHelpers.Furlong +using Main.TestHelpers.Furlong @testset "Unitful elements" begin r = Furlong(1):Furlong(1):Furlong(2) a = collect(r)