diff --git a/base/loading.jl b/base/loading.jl index 7d2f33ede633e..33799805e0601 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2625,8 +2625,7 @@ function __require_prelocked(pkg::PkgId, env) if JLOptions().use_compiled_modules == 1 if !generating_output(#=incremental=#false) project = active_project() - if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) && project !== nothing && - isfile(project) && project_file_manifest_path(project) !== nothing + if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) parallel_precompile_attempted = true unlock(require_lock) try diff --git a/base/precompilation.jl b/base/precompilation.jl index e737faa210c79..1b609f3e2015a 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -26,9 +26,24 @@ struct ExplicitEnv #local_prefs::Union{Nothing, Dict{String, Any}} end -function ExplicitEnv(envpath::String=Base.active_project()) +ExplicitEnv() = ExplicitEnv(Base.active_project()) +function ExplicitEnv(::Nothing, envpath::String="") + ExplicitEnv(envpath, + Dict{String, UUID}(), # project_deps + Dict{String, UUID}(), # project_weakdeps + Dict{String, UUID}(), # project_extras + Dict{String, Vector{UUID}}(), # project_extensions + Dict{UUID, Vector{UUID}}(), # deps + Dict{UUID, Vector{UUID}}(), # weakdeps + Dict{UUID, Dict{String, Vector{UUID}}}(), # extensions + Dict{UUID, String}(), # names + Dict{UUID, Union{SHA1, String, Nothing, Missing}}()) +end +function ExplicitEnv(envpath::String) + # Handle missing project file by creating an empty environment if !isfile(envpath) - error("expected a project file at $(repr(envpath))") + envpath = abspath(envpath) + return ExplicitEnv(nothing, envpath) end envpath = abspath(envpath) project_d = parsed_toml(envpath) @@ -468,6 +483,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; fancyprint::Bool = can_fancyprint(io) && !timing, manifest::Bool=false, ignore_loaded::Bool=true) + @debug "precompilepkgs called with" pkgs internal_call strict warn_loaded timing _from_loading configs fancyprint manifest ignore_loaded # monomorphize this to avoid latency problems _precompilepkgs(pkgs, internal_call, strict, warn_loaded, timing, _from_loading, configs isa Vector{Config} ? configs : [configs], @@ -518,9 +534,12 @@ function _precompilepkgs(pkgs::Vector{String}, # inverse map of `parent_to_ext` above (ext → parent) ext_to_parent = Dict{Base.PkgId, Base.PkgId}() - function describe_pkg(pkg::PkgId, is_project_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) + function describe_pkg(pkg::PkgId, is_project_dep::Bool, is_serial_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) name = full_name(ext_to_parent, pkg) name = is_project_dep ? name : color_string(name, :light_black) + if is_serial_dep + name *= color_string(" (serial)", :light_black) + end if nconfigs > 1 && !isempty(flags) config_str = join(flags, " ") name *= color_string(" `$config_str`", :light_black) @@ -630,15 +649,28 @@ function _precompilepkgs(pkgs::Vector{String}, end @debug "precompile: extensions collected" + serial_deps = Base.PkgId[] # packages that are being precompiled in serial + + if _from_loading + # if called from loading precompilation it may be a package from another environment stack + # where we don't have access to the dep graph, so just add as a single package and do serial + # precompilation of its deps within the job. + for pkg in requested_pkgs # In case loading asks for multiple packages + pkgid = Base.identify_package(pkg) + pkgid === nothing && continue + if !haskey(direct_deps, pkgid) + @debug "precompile: package `$(pkgid)` is outside of the environment, so adding as single package serial job" + direct_deps[pkgid] = Base.PkgId[] # no deps, do them in serial in the job + push!(project_deps, pkgid) # add to project_deps so it doesn't show up in gray + push!(serial_deps, pkgid) + end + end + end + # return early if no deps if isempty(direct_deps) if isempty(pkgs) return - elseif _from_loading - # if called from loading precompilation it may be a package from another environment stack so - # don't error and allow serial precompilation to try - # TODO: actually handle packages from other envs in the stack - return else error("No direct dependencies outside of the sysimage found matching $(pkgs)") end @@ -846,7 +878,7 @@ function _precompilepkgs(pkgs::Vector{String}, dep, config = pkg_config loaded = warn_loaded && haskey(Base.loaded_modules, dep) flags, cacheflags = config - name = describe_pkg(dep, dep in project_deps, flags, cacheflags) + name = describe_pkg(dep, dep in project_deps, dep in serial_deps, flags, cacheflags) line = if pkg_config in precomperr_deps string(color_string(" ? ", Base.warn_color()), name) elseif haskey(failed_deps, pkg_config) @@ -929,12 +961,13 @@ function _precompilepkgs(pkgs::Vector{String}, if !circular && is_stale Base.acquire(parallel_limiter) is_project_dep = pkg in project_deps + is_serial_dep = pkg in serial_deps # std monitoring std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) - name = describe_pkg(pkg, is_project_dep, flags, cacheflags) + name = describe_pkg(pkg, is_project_dep, is_serial_dep, flags, cacheflags) @lock print_lock begin if !fancyprint && isempty(pkg_queue) printpkgstyle(io, :Precompiling, something(target[], "packages...")) diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 567b4699c3cf6..301f5e5def618 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -42,6 +42,9 @@ These environments each serve a different purpose: * Package directories provide **convenience** when a full carefully-tracked project environment is unnecessary. They are useful when you want to put a set of packages somewhere and be able to directly use them, without needing to create a project environment for them. * Stacked environments allow for **adding** tools to the primary environment. You can push an environment of development tools onto the end of the stack to make them available from the REPL and scripts, but not from inside packages. +!!! note + When loading a package from another environment in the stack other than the active environment the package is loaded in the context of the active environment. This means that the package will be loaded as if it were imported in the active environment, which may affect how its dependencies versions are resolved. When such a package is precompiling it will be marked as a `(serial)` precompile job, which means that its dependencies will be precompiled in series within the same job, which will likely be slower. + At a high-level, each environment conceptually defines three maps: roots, graph and paths. When resolving the meaning of `import X`, the roots and graph maps are used to determine the identity of `X`, while the paths map is used to locate the source code of `X`. The specific roles of the three maps are: - **roots:** `name::Symbol` ⟶ `uuid::UUID` diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index 34ef9a796ba56..0744cac679698 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -32,5 +32,5 @@ end @test lines[9] == "called bar" @test lines[10] == "calling new bar" @test lines[11] == " From worker 2:\tTaking over the world..." - @test readline(err) == "exception caught from C" + @test "exception caught from C" in readlines(err) end diff --git a/test/loading.jl b/test/loading.jl index e95138e27f4dc..3a085277d96b8 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1495,7 +1495,7 @@ end # helper function to load a package and return the output function load_package(name, args=``) - code = "using $name" + code = "Base.disable_parallel_precompile = true; using $name" cmd = addenv(`$(Base.julia_cmd()) -e $code $args`, "JULIA_LOAD_PATH" => dir, "JULIA_DEPOT_PATH" => depot_path, diff --git a/test/precompile.jl b/test/precompile.jl index d1ad0c459932d..8e091692b68fd 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -686,16 +686,15 @@ precompile_test_harness(false) do dir error("break me") end """) - @test_warn r"LoadError: break me\nStacktrace:\n[ ]*\[1\] [\e01m\[]*error" try - Base.require(Main, :FooBar2) - error("the \"break me\" test failed") - catch exc - isa(exc, ErrorException) || rethrow() - # The LoadError shouldn't be surfaced but is printed to stderr, hence the `@test_warn` capture tests - occursin("LoadError: break me", exc.msg) && rethrow() - # The actual error that is thrown - occursin("Failed to precompile FooBar2", exc.msg) || rethrow() - end + try + Base.require(Main, :FooBar2) + error("the \"break me\" test failed") + catch exc + isa(exc, Base.Precompilation.PkgPrecompileError) || rethrow() + occursin("Failed to precompile FooBar2", exc.msg) || rethrow() + # The LoadError is printed to stderr in the precompilepkgs worker and captured in the PkgPrecompileError msg + occursin("LoadError: break me", exc.msg) || rethrow() + end # Test that trying to eval into closed modules during precompilation is an error FooBar3_file = joinpath(dir, "FooBar3.jl") @@ -707,11 +706,12 @@ precompile_test_harness(false) do dir $code end """) - @test_warn "Evaluation into the closed module `Base` breaks incremental compilation" try - Base.require(Main, :FooBar3) - catch exc - isa(exc, ErrorException) || rethrow() - end + try + Base.require(Main, :FooBar3) + catch exc + isa(exc, Base.Precompilation.PkgPrecompileError) || rethrow() + occursin("Evaluation into the closed module `Base` breaks incremental compilation", exc.msg) || rethrow() + end end # Test transitive dependency for #21266