Merge pull request #189 from JuliaCI/tb/artifacts

Take rootfs artifact extraction into our own hands

Artifacts.toml

@@ -14,12 +14,12 @@ git-tree-sha1 = "795db6086ddceb415b9dbd8a1f257836d05ac757"
     url = "https://github.com/JuliaCI/PkgEval.jl/releases/download/v0.1/arch-devel-20220628.tar.xz"
 
 [debian]
-git-tree-sha1 = "d41ccdb32c924682293a4fe525b9986b7debb11d"
+git-tree-sha1 = "db0b80b8f274edfbd13651cceb308df81925e974"
 lazy = true
 
     [[debian.download]]
-    sha256 = "072392d8635847de70665af512551038b04baca99b38e271d0eef1ea36d2833a"
-    url = "https://github.com/JuliaCI/PkgEval.jl/releases/download/v0.1/debian-bullseye-20220818.tar.xz"
+    sha256 = "a5a1c9c7ea94e2036634978413a919fe97591c09983c45eb9ce5936b05e1c1c4"
+    url = "https://github.com/JuliaCI/PkgEval.jl/releases/download/v0.1/debian-bullseye-20221202.tar.xz"
 
 [package_linux]
 git-tree-sha1 = "104eaa2e0f648fd5de41697329b544a92f3edabf"

Project.toml

@@ -4,6 +4,9 @@ authors = ["Keno Fischer <[email protected]>", "Tim Besard <tim@juliacompu
 version = "0.2.0"
 
 [deps]
+CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
@@ -18,6 +21,7 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Sandbox = "9307e30f-c43e-9ca7-d17c-c2dc59df670d"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 s5cmd_jll = "9c7924da-da29-52d4-adf6-81e6cfbf5c74"
 
 [compat]

src/PkgEval.jl

@@ -25,6 +25,7 @@ include("utils.jl")
 function __init__()
     global download_dir = @get_scratch!("downloads")
     mkpath(joinpath(download_dir, "srccache"))
+    mkpath(joinpath(download_dir, "rootfs"))
 
     global storage_dir = @get_scratch!("storage")
     mkpath(joinpath(storage_dir, "artifacts"))

src/rootfs.jl

@@ -1,13 +1,45 @@
 using LazyArtifacts: @artifact_str
-
 lazy_artifact(x) = @artifact_str(x)
 
-function _create_rootfs(config::Configuration)
-    base = lazy_artifact(config.rootfs)
+using Tar: extract
+using CodecZlib: GzipDecompressorStream
+using CodecXz: XzDecompressorStream
+using CodecZstd: ZstdDecompressorStream
 
+function _create_rootfs(config::Configuration)
     # a bare rootfs isn't usable out-of-the-box
     derived = mktempdir(prefix="pkgeval_rootfs_")
-    cp(base, derived; force=true)
+
+    # BUG: Pkg installs our artifacts with wrong permissions (JuliaLang/Pkg.jl#/3269)
+    if false
+        base = lazy_artifact(config.rootfs)
+        cp(base, derived; force=true)
+    else
+        artifacts_toml = TOML.parsefile(joinpath(dirname(@__DIR__), "Artifacts.toml"))
+        rootfs_toml = artifacts_toml[config.rootfs]
+
+        # download
+        url = rootfs_toml["download"][1]["url"]
+        hash = rootfs_toml["download"][1]["sha256"]
+        tarball = joinpath(download_dir, "rootfs", basename(url))
+        if !isfile(tarball)
+            Pkg.PlatformEngines.download_verify(url, hash, tarball)
+        end
+
+        # extract
+        open(tarball) do io
+            stream = if endswith(tarball, ".xz")
+                XzDecompressorStream(io)
+            elseif endswith(tarball, ".gz")
+                GzipDecompressorStream(io)
+            elseif endswith(tarball, ".zst")
+                ZstdDecompressorStream(io)
+            else
+                error("Unknown file extension")
+            end
+            extract(stream, derived)
+        end
+    end
 
     # add a user and group
     chmod(joinpath(derived, "etc/passwd"), 0o644)