diff --git a/base/loading.jl b/base/loading.jl index 33799805e0601..628795da6aa8e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2629,7 +2629,7 @@ function __require_prelocked(pkg::PkgId, env) parallel_precompile_attempted = true unlock(require_lock) try - Precompilation.precompilepkgs([pkg.name]; _from_loading=true, ignore_loaded=false) + Precompilation.precompilepkgs([pkg]; _from_loading=true, ignore_loaded=false) finally lock(require_lock) end diff --git a/base/precompilation.jl b/base/precompilation.jl index 1b609f3e2015a..45634d877115f 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -41,7 +41,7 @@ function ExplicitEnv(::Nothing, envpath::String="") end function ExplicitEnv(envpath::String) # Handle missing project file by creating an empty environment - if !isfile(envpath) + if !isfile(envpath) || project_file_manifest_path(envpath) === nothing envpath = abspath(envpath) return ExplicitEnv(nothing, envpath) end @@ -471,7 +471,7 @@ function collect_all_deps(direct_deps, dep, alldeps=Set{Base.PkgId}()) end -function precompilepkgs(pkgs::Vector{String}=String[]; +function precompilepkgs(pkgs::Union{Vector{String}, Vector{PkgId}}=String[]; internal_call::Bool=false, strict::Bool = false, warn_loaded::Bool = true, @@ -490,7 +490,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; IOContext{IO}(io), fancyprint, manifest, ignore_loaded) end -function _precompilepkgs(pkgs::Vector{String}, +function _precompilepkgs(pkgs::Union{Vector{String}, Vector{PkgId}}, internal_call::Bool, strict::Bool, warn_loaded::Bool, @@ -502,6 +502,23 @@ function _precompilepkgs(pkgs::Vector{String}, manifest::Bool, ignore_loaded::Bool) requested_pkgs = copy(pkgs) # for understanding user intent + pkg_names = pkgs isa Vector{String} ? copy(pkgs) : String[pkg.name for pkg in pkgs] + if pkgs isa Vector{PkgId} + requested_pkgids = copy(pkgs) + else + requested_pkgids = PkgId[] + for name in pkgs + pkgid = Base.identify_package(name) + if pkgid === nothing + if _from_loading + return # leave it up to loading to handle this + else + throw(PkgPrecompileError("Unknown package: $name")) + end + end + push!(requested_pkgids, pkgid) + end + end time_start = time_ns() @@ -655,8 +672,7 @@ function _precompilepkgs(pkgs::Vector{String}, # if called from loading precompilation it may be a package from another environment stack # where we don't have access to the dep graph, so just add as a single package and do serial # precompilation of its deps within the job. - for pkg in requested_pkgs # In case loading asks for multiple packages - pkgid = Base.identify_package(pkg) + for pkgid in requested_pkgids # In case loading asks for multiple packages pkgid === nothing && continue if !haskey(direct_deps, pkgid) @debug "precompile: package `$(pkgid)` is outside of the environment, so adding as single package serial job" @@ -704,6 +720,7 @@ function _precompilepkgs(pkgs::Vector{String}, circular_deps = Base.PkgId[] for pkg in keys(direct_deps) @assert isempty(stack) + pkg in serial_deps && continue # skip serial deps as we don't have their dependency graph if scan_pkg!(stack, could_be_cycle, cycles, pkg, direct_deps) push!(circular_deps, pkg) for pkg_config in keys(was_processed) @@ -717,18 +734,31 @@ function _precompilepkgs(pkgs::Vector{String}, end @debug "precompile: circular dep check done" + # If you have a workspace and want to precompile all projects in it, look through all packages in the manifest + # instead of collecting from a project i.e. not filter out packages that are in the current project. + # i.e. Pkg sets manifest to true for workspace precompile requests + # TODO: rename `manifest`? if !manifest - if isempty(pkgs) - pkgs = [pkg.name for pkg in project_deps] + if isempty(pkg_names) + pkg_names = [pkg.name for pkg in project_deps] end keep = Set{Base.PkgId}() for dep in direct_deps dep_pkgid = first(dep) - if dep_pkgid.name in pkgs + if dep_pkgid.name in pkg_names push!(keep, dep_pkgid) collect_all_deps(direct_deps, dep_pkgid, keep) end end + # Also keep packages that were explicitly requested as PkgIds (for extensions) + if pkgs isa Vector{PkgId} + for requested_pkgid in requested_pkgids + if haskey(direct_deps, requested_pkgid) + push!(keep, requested_pkgid) + collect_all_deps(direct_deps, requested_pkgid, keep) + end + end + end for ext in keys(ext_to_parent) if issubset(collect_all_deps(direct_deps, ext), keep) # if all extension deps are kept push!(keep, ext) @@ -936,7 +966,8 @@ function _precompilepkgs(pkgs::Vector{String}, for (pkg, deps) in direct_deps cachepaths = get!(() -> Base.find_all_in_cache_path(pkg), cachepath_cache, pkg) sourcepath = Base.locate_package(pkg) - single_requested_pkg = length(requested_pkgs) == 1 && only(requested_pkgs) == pkg.name + single_requested_pkg = length(requested_pkgs) == 1 && + (pkg in requested_pkgids || pkg.name in pkg_names) for config in configs pkg_config = (pkg, config) if sourcepath === nothing @@ -1063,7 +1094,7 @@ function _precompilepkgs(pkgs::Vector{String}, str = sprint(context=io) do iostr if !quick_exit if fancyprint # replace the progress bar - what = isempty(requested_pkgs) ? "packages finished." : "$(join(requested_pkgs, ", ", " and ")) finished." + what = isempty(requested_pkgids) ? "packages finished." : "$(join((p.name for p in requested_pkgids), ", ", " and ")) finished." printpkgstyle(iostr, :Precompiling, what) end plural = length(configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" diff --git a/test/precompile.jl b/test/precompile.jl index 8e091692b68fd..cce7e25caed79 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -2496,6 +2496,46 @@ precompile_test_harness("Package top-level load itself") do load_path end end +precompile_test_harness("Package precompilation works without manifest") do load_path + pkg_dir = joinpath(load_path, "TestPkgNoManifest") + mkpath(pkg_dir) + + # Create Project.toml with stdlib dependencies + write(joinpath(pkg_dir, "Project.toml"), """ + name = "TestPkgNoManifest" + uuid = "f47a8e44-5f82-4c5c-9076-4b4e8b7e8e8e" + version = "0.1.0" + + [deps] + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + """) + + # Create src directory and main module file + src_dir = joinpath(pkg_dir, "src") + mkpath(src_dir) + write(joinpath(src_dir, "TestPkgNoManifest.jl"), """ + module TestPkgNoManifest + end + """) + + old_active_project = Base.active_project() + try + # Activate the new package environment + Base.set_active_project(joinpath(pkg_dir, "Project.toml")) + + # Ensure there's no manifest file (this is the key to the test) + manifest_path = joinpath(pkg_dir, "Manifest.toml") + isfile(manifest_path) && rm(manifest_path) + + # This should work without errors - precompiling a package with no manifest + @eval using TestPkgNoManifest + finally + # Restore original load path and active project + Base.set_active_project(old_active_project) + end +end + # Verify that inference / caching was not performed for any macros in the sysimage let m = only(methods(Base.var"@big_str")) @test m.specializations === Core.svec() || !isdefined(m.specializations, :cache)