From 0a0922ea9f3722b89aa50176a99f6b854ca4713e Mon Sep 17 00:00:00 2001 From: KristofferC Date: Mon, 28 Oct 2024 15:39:11 +0100 Subject: [PATCH 1/3] cache the `find_all_in_cache_path` call during parallel precompilation --- base/loading.jl | 10 ++++++---- base/precompilation.jl | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 6391e2511f8d5..b493a294dc509 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1816,7 +1816,8 @@ const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String} function compilecache_path(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), - cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), + cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(), + cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(modkey), cachepath_cache, pkg), sourcepath::Union{String,Nothing}=Base.locate_package(pkg), flags::CacheFlags=CacheFlags()) path = nothing @@ -1831,7 +1832,7 @@ function compilecache_path(pkg::PkgId; for dep in staledeps dep isa Module && continue modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} - modpaths = find_all_in_cache_path(modkey) + modpaths = get!(() -> find_all_in_cache_path(modkey), cachepath_cache, modkey) for modpath_to_try in modpaths::Vector{String} stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded, requested_flags=flags) === true, @@ -1873,10 +1874,11 @@ fresh julia session specify `ignore_loaded=true`. function isprecompiled(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), - cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), + cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(), + cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(modkey), cachepath_cache, pkg), sourcepath::Union{String,Nothing}=Base.locate_package(pkg), flags::CacheFlags=CacheFlags()) - path = compilecache_path(pkg; ignore_loaded, stale_cache, cachepaths, sourcepath, flags) + path = compilecache_path(pkg; ignore_loaded, stale_cache, cachepath_cache, cachepaths, sourcepath, flags) return !isnothing(path) end diff --git a/base/precompilation.jl b/base/precompilation.jl index f597acef9b57f..386e8a7a6c404 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -415,6 +415,8 @@ function _precompilepkgs(pkgs::Vector{String}, color_string(cstr::String, col::Union{Int64, Symbol}) = _color_string(cstr, col, hascolor) stale_cache = Dict{StaleCacheKey, Bool}() + cachepath_cache = Dict{PkgId, Vector{String}}() + exts = Dict{PkgId, String}() # ext -> parent # make a flat map of each dep and its direct deps depsmap = Dict{PkgId, Vector{PkgId}}() @@ -785,7 +787,7 @@ function _precompilepkgs(pkgs::Vector{String}, ## precompilation loop for (pkg, deps) in depsmap - cachepaths = Base.find_all_in_cache_path(pkg) + cachepaths = get!(() -> Base.find_all_in_cache_path(pkg), cachepath_cache, pkg) sourcepath = Base.locate_package(pkg) single_requested_pkg = length(requested_pkgs) == 1 && only(requested_pkgs) == pkg.name for config in configs @@ -808,7 +810,7 @@ function _precompilepkgs(pkgs::Vector{String}, wait(was_processed[(dep,config)]) end circular = pkg in circular_deps - is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) + is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepath_cache, cachepaths, sourcepath, flags=cacheflags) if !circular && is_stale Base.acquire(parallel_limiter) is_direct_dep = pkg in direct_deps From af8c58f76b217321f135b0d3e3f381025aaa8d48 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 28 Oct 2024 22:22:32 +0100 Subject: [PATCH 2/3] Update base/loading.jl Co-authored-by: Ian Butterworth --- base/loading.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index b493a294dc509..4f4c66101b0fb 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1875,7 +1875,7 @@ function isprecompiled(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(), - cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(modkey), cachepath_cache, pkg), + cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg), sourcepath::Union{String,Nothing}=Base.locate_package(pkg), flags::CacheFlags=CacheFlags()) path = compilecache_path(pkg; ignore_loaded, stale_cache, cachepath_cache, cachepaths, sourcepath, flags) From cf3b6bda756aaa4c0a7e3171a4b7619c68c05a8e Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 28 Oct 2024 22:52:50 -0400 Subject: [PATCH 3/3] Update loading.jl --- base/loading.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index 4f4c66101b0fb..5eda281fb02a1 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1817,7 +1817,7 @@ function compilecache_path(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(), - cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(modkey), cachepath_cache, pkg), + cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg), sourcepath::Union{String,Nothing}=Base.locate_package(pkg), flags::CacheFlags=CacheFlags()) path = nothing