Disable generation of pkgimages. (#237)

Also reworks how the test harness is invoked, now passing full Configuration
and Package objects for easier configuration of the sandbox.

Project.toml

@@ -19,6 +19,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 crun_jll = "aef68bef-9b3f-58c3-affd-5589e119eed3"

scripts/common.jl

@@ -0,0 +1,36 @@
+include("../src/PkgEvalCore.jl")
+using .PkgEvalCore
+
+using Pkg
+using Base: UUID
+
+# simplified version of utilities from utils.jl (with no need to
+# scan for children, as we use this from the parent when idle)
+function cpu_time()
+    stats = read("/proc/self/stat", String)
+
+    m = match(r"^(\d+) \((.+)\) (.+)", stats)
+    @assert m !== nothing "Invalid contents for /proc/self/stat: $stats"
+    fields = [[m.captures[1], m.captures[2]]; split(m.captures[3])]
+    utime = parse(Int, fields[14])
+    stime = parse(Int, fields[15])
+    cutime = parse(Int, fields[16])
+    cstime = parse(Int, fields[17])
+
+    return (utime + stime + cutime + cstime) / Sys.SC_CLK_TCK
+end
+function io_bytes()
+    stats = read("/proc/self/io", String)
+
+    dict = Dict()
+    for line in split(stats, '\n')
+        m = match(r"^(.+): (\d+)$", line)
+        m === nothing && continue
+        dict[m.captures[1]] = parse(Int, m.captures[2])
+    end
+
+    return dict["rchar"] + dict["wchar"]
+end
+
+using Dates
+elapsed(t) = "$(round(cpu_time() - t; digits=2))s"

scripts/compile.jl

@@ -0,0 +1,44 @@
+include("common.jl")
+
+config = eval(Meta.parse(ARGS[1]))
+pkg = eval(Meta.parse(ARGS[2]))
+sysimage_path = ARGS[3]
+
+println("Package compilation of $(pkg.name) started at ", now(UTC))
+
+println()
+using InteractiveUtils
+versioninfo()
+
+
+print("\n\n", '#'^80, "\n# Installation\n#\n\n")
+t0 = time()
+
+is_stdlib = any(Pkg.Types.stdlibs()) do (uuid,stdlib)
+    name = isa(stdlib, String) ? stdlib : first(stdlib)
+    name == pkg.name
+end
+is_stdlib && error("Packages that are standard libraries cannot be compiled again.")
+
+println("Installing PackageCompiler...")
+project = Base.active_project()
+Pkg.activate(; temp=true)
+Pkg.add(name="PackageCompiler", uuid="9b87118b-4619-50d2-8e1e-99f35a4d4d9d")
+using PackageCompiler
+Pkg.activate(project)
+
+println("\nInstalling $(pkg.name)...")
+
+Pkg.add(convert(Pkg.Types.PackageSpec, pkg))
+
+println("\nCompleted after $(elapsed(t0))")
+
+
+print("\n\n", '#'^80, "\n# Compilation\n#\n\n")
+t1 = time()
+
+create_sysimage([pkg.name]; sysimage_path)
+println("\nCompleted after $(elapsed(t1))")
+
+s = stat(sysimage_path).size
+println("Generated system image is ", Base.format_bytes(s))

scripts/report_bug.jl

@@ -0,0 +1,25 @@
+include("common.jl")
+
+config = eval(Meta.parse(ARGS[1]))
+pkg = eval(Meta.parse(ARGS[2]))
+
+print("\n\n", '#'^80, "\n# Bug reporting\n#\n\n")
+t0 = cpu_time()
+
+try
+    # use a clean environment, or BugReporting's deps could
+    # affect/be affected by the tested package's dependencies.
+    Pkg.activate(; temp=true)
+    Pkg.add(name="BugReporting", uuid="bcf9a6e7-4020-453c-b88e-690564246bb8")
+    using BugReporting
+
+    trace_dir = BugReporting.default_rr_trace_dir()
+    trace = BugReporting.find_latest_trace(trace_dir)
+    BugReporting.compress_trace(trace, "/output/$(pkg.name).tar.zst")
+    println("\nBugReporting completed after $(elapsed(t0))")
+catch err
+    println("\nBugReporting failed after $(elapsed(t0))")
+    showerror(stdout, err)
+    Base.show_backtrace(stdout, catch_backtrace())
+    println()
+end

scripts/test.jl

@@ -0,0 +1,161 @@
+include("common.jl")
+
+config = eval(Meta.parse(ARGS[1]))
+pkg = eval(Meta.parse(ARGS[2]))
+
+println("Package evaluation of $(pkg.name) started at ", now(UTC))
+
+println()
+using InteractiveUtils
+versioninfo()
+
+
+print("\n\n", '#'^80, "\n# Set-up\n#\n\n")
+
+# we install PkgEval dependencies in a separate environment
+Pkg.activate("pkgeval"; shared=true)
+
+deps = ["TestEnv"]
+
+if config.rr == RREnabled
+    push!(deps, "BugReporting")
+
+    # instead of using --bug-report, we'll load BugReporting manually, because
+    # by default we can't access any unrelated package from within the sandbox
+    # created by Pkg, resulting in re-installation and compilation of BugReporting.
+    open("bugreport.jl", "w") do io
+        # loading an unrelated package like this is normally a bad thing to do,
+        # because the versions of BugReporting.jl's dependencies may conflict with
+        # the dependencies of the package under evaluation. However, in the session
+        # where we load BugReporting.jl we'll never actually load the package we
+        # want to test, only re-start Julia under rr, so this should be fine.
+        println(io, "pushfirst!(LOAD_PATH, $(repr(Base.ACTIVE_PROJECT[])))")
+
+        # this code is essentially what --bug-report from InteractiveUtils does
+        println(io, "using BugReporting")
+        println(io, "println(\"Switching execution to under rr\")")
+        println(io, "BugReporting.make_interactive_report(\"rr-local\", ARGS)")
+    end
+end
+
+# generating package images is really expensive, without much benefit (for PkgEval)
+# so determine here if we need to disable them using additional CLI args
+# (we can't do this externally because of JuliaLang/Pkg.jl#3737)
+julia_args = if VERSION < v"1.9-beta1" || (v"1.10-" <= VERSION < v"1.10.0-DEV.204")
+    # we don't support pkgimages yet
+    ``
+elseif any(startswith("--pkgimages"), config.julia_flags)
+    # the user specifically requested pkgimages
+    ``
+else
+    if VERSION >= v"1.11-DEV.1119"
+        # we can selectively disable pkgimages while allowing reuse of existing ones
+        `--pkgimages=existing`
+    elseif VERSION >= v"1.11-DEV.123"
+        # we can only selectively disable all compilation caches. this isn't ideal,
+        # but at this point in time (where many stdlibs have been moved out of the
+        # system image) it's strictly better than using `--pkgimages=no`
+        `--compiled-modules=existing`
+    else
+        # completely disable pkgimages
+        `--pkgimages=no`
+    end
+end
+
+Pkg.add(deps)
+
+Pkg.activate()
+
+
+print("\n\n", '#'^80, "\n# Installation\n#\n\n")
+
+t0 = cpu_time()
+try
+    Pkg.add(convert(Pkg.Types.PackageSpec, pkg))
+
+    println("\nInstallation completed after $(elapsed(t0))")
+    write("/output/installed", repr(true))
+catch
+    println("\nInstallation failed after $(elapsed(t0))\n")
+    write("/output/installed", repr(false))
+    rethrow()
+finally
+    # even if a package fails to install, it may have been resolved
+    # (e.g., when the build phase errors)
+    for package_info in values(Pkg.dependencies())
+        if package_info.name == pkg.name
+            write("/output/version", repr(package_info.version))
+            break
+        end
+    end
+end
+
+
+if config.precompile
+print("\n\n", '#'^80, "\n# Precompilation\n#\n\n")
+
+# we run with JULIA_PKG_PRECOMPILE_AUTO=0 to avoid precompiling on Pkg.add,
+# because we can't use the generated images for Pkg.test which uses different
+# options (i.e., --check-bounds=yes). however, to get accurate test timings,
+# we *should* precompile before running tests, so we do that here manually.
+
+t0 = cpu_time()
+try
+    run(```$(Base.julia_cmd()) $(julia_args) --check-bounds=yes
+            -e 'using Pkg
+
+                Pkg.activate("pkgeval"; shared=true)
+
+                # precompile PkgEval run-time dependencies (notably BugReporting.jl)
+                Pkg.precompile()
+
+                # try to use TestEnv to precompile the package test dependencies
+                try
+                    using TestEnv
+                    Pkg.activate()
+                    TestEnv.activate(ARGS[1])
+                catch err
+                    @error "Failed to use TestEnv.jl; test dependencies will not be precompiled" exception=(err, catch_backtrace())
+                    Pkg.activate()
+                end
+                Pkg.precompile()' $(pkg.name)```)
+
+    println("\nPrecompilation completed after $(elapsed(t0))")
+catch
+    println("\nPrecompilation failed after $(elapsed(t0))\n")
+end
+end
+
+
+print("\n\n", '#'^80, "\n# Testing\n#\n\n")
+
+is_stdlib = any(Pkg.Types.stdlibs()) do (uuid,stdlib)
+    name = isa(stdlib, String) ? stdlib : first(stdlib)
+    name == pkg.name
+end
+if is_stdlib
+    println("\n$(pkg.name) is a standard library in this Julia build.")
+
+    # we currently only support testing the embedded version of stdlib packages
+    if pkg.version !== nothing || pkg.url !== nothing || pkg.rev !== nothing
+        error("Packages that are standard libraries can only be tested using the embedded version.")
+    end
+end
+
+t0 = cpu_time()
+io0 = io_bytes()
+try
+    if config.rr == RREnabled
+        Pkg.test(pkg.name; julia_args=`$julia_args --load bugreport.jl`)
+    else
+        Pkg.test(pkg.name; julia_args)
+    end
+
+    println("\nTesting completed after $(elapsed(t0))")
+catch
+    println("\nTesting failed after $(elapsed(t0))\n")
+    rethrow()
+finally
+    write("/output/duration", repr(cpu_time()-t0))
+    write("/output/input_output", repr(io_bytes()-io0))
+end

src/PkgEval.jl

@@ -4,6 +4,7 @@ using Pkg, LazyArtifacts, Random
 import Pkg.TOML
 import GitHub
 using Base: UUID
+using Reexport: @reexport
 
 import Scratch: @get_scratch!
 download_dir = ""
@@ -23,7 +24,9 @@ slow_list = String[]
 # and not reuse, e.g., when running in a testset
 const rng = MersenneTwister()
 
-include("types.jl")
+include("PkgEvalCore.jl")
+@reexport using .PkgEvalCore
+
 include("registry.jl")
 include("rootfs.jl")
 include("buildkite.jl")

src/types.jl → src/PkgEvalCore.jl

@@ -1,4 +1,11 @@
-export Configuration, ismodified, Package
+module PkgEvalCore
+
+using Pkg
+using Base: UUID
+
+export Configuration, ismodified, Package,
+       package_spec_tuple, can_use_binaries,
+       RREnabled, RREnabledOnRetry, RRDisabled
 
 
 ## individual settings, keeping track of modifications
@@ -91,7 +98,24 @@ Base.@kwdef struct Configuration
     compile_time_limit::Setting{Float64} = Default(30*60) # 30 mins
 end
 
-function Base.show(io::IO, cfg::Configuration)
+function durationstring(seconds)
+    components = String[]
+    divisions = [3600 => "hour", 60 => "min", 1 => "sec"]
+    for (div, name) in divisions
+        if seconds >= div
+            push!(components, "$(Int(seconds ÷ div)) $name")
+            seconds %= div
+        end
+    end
+    if isempty(components)
+        "0 sec"
+    else
+        join(components, " ")
+    end
+end
+
+# verbose printing
+function Base.show(io::IO, ::MIME"text/plain", cfg::Configuration)
     function show_setting(field, renderer=identity)
         setting = getfield(cfg, Symbol(field))
         value_str = renderer(setting[])
@@ -126,6 +150,21 @@ function Base.show(io::IO, cfg::Configuration)
     print(io, ")")
 end
 
+# compact printing, making sure the output is parseable again
+function Base.show(io::IO, cfg::Configuration)
+    default_cfg = Configuration()
+    print(io, "Configuration(")
+    got_fields = false
+    for field in fieldnames(Configuration)
+        if getproperty(cfg, field) != getproperty(default_cfg, field)
+            got_fields && print(io, ", ")
+            print(io, field, "=", repr(getproperty(cfg, field)))
+            got_fields = true
+        end
+    end
+    print(io, ")")
+end
+
 # when requested, return the underlying value
 function Base.getproperty(cfg::Configuration, field::Symbol)
     val = getfield(cfg, field)
@@ -188,23 +227,30 @@ function Package(cfg::Package; kwargs...)
 end
 
 # convert a Package to a tuple that's Pkg.add'able
-function package_spec_tuple(pkg::Package)
+function Base.convert(::Type{Pkg.Types.PackageSpec}, pkg::Package)
     spec = (;)
     for field in (:name, :uuid, :version, :url, :rev)
         val = getfield(pkg, field)
         if val !== nothing
             spec = merge(spec, NamedTuple{(field,)}((val,)))
         end
     end
-    spec
+    PackageSpec(; spec...)
 end
 
+# compact printing, making sure the output is parseable again
+function Base.show(io::IO, pkg::Package)
+    print(io, "Package(")
+    got_fields = false
+    for field in fieldnames(Package)
+        val = getfield(pkg, field)
+        if val !== nothing
+            got_fields && print(io, ", ")
+            print(io, field, "=", repr(val))
+            got_fields = true
+        end
+    end
+    print(io, ")")
+end
 
-## test job
-
-struct Job
-    config::Configuration
-    package::Package
-
-    use_cache::Bool
 end