From 841b0686826d78636009ae22e953680f634b7d31 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Wed, 5 Feb 2020 13:19:43 -0800 Subject: [PATCH] Use `Pkg.BinaryPlatforms` and `Pkg.PlatformEngines` to do the heavy lifting Now that `Pkg` has inlined the majority of the work that BP was doing, it doesn't make sense to maintain multiple copies of the same code; instead, just have `BP` wrap around `Pkg` for compatibility, and push everyone to migrate to Artifacts. --- .cirrus.yml | 3 +- .travis.yml | 17 +- Project.toml | 8 +- appveyor.yml | 2 +- src/BinaryProvider.jl | 25 +- src/CompatShims.jl | 20 - src/PlatformEngines.jl | 906 ----------------------------------- src/PlatformNames.jl | 758 ----------------------------- src/Prefix.jl | 170 +------ src/precompile.jl | 49 -- test/LibFoo.jl/deps/build.jl | 20 +- test/runtests.jl | 300 ++---------- 12 files changed, 69 insertions(+), 2209 deletions(-) delete mode 100644 src/CompatShims.jl delete mode 100644 src/PlatformEngines.jl delete mode 100644 src/PlatformNames.jl diff --git a/.cirrus.yml b/.cirrus.yml index c9a2aaa..d7ba1cf 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,8 +4,7 @@ task: name: FreeBSD env: matrix: - - JULIA_VERSION: 1.0 - - JULIA_VERSION: 1.1 + - JULIA_VERSION: 1.3 - JULIA_VERSION: nightly install_script: - sh -c "$(fetch https://raw.githubusercontent.com/ararslan/CirrusCI.jl/master/bin/install.sh -o -)" diff --git a/.travis.yml b/.travis.yml index 6c42a22..f862892 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,26 +4,11 @@ os: - linux - osx julia: - - 0.7 - - 1.0 + - 1.3 - nightly notifications: email: false -matrix: - include: - # Make sure to override to "wget" at least once - - julia: 1.0 - os: linux - env: BINARYPROVIDER_DOWNLOAD_ENGINE="wget" - - # Test with `busybox` tools - # NOTE: This disabled for now, as Ubuntu's busybox doesn't support HTTPS. - # Tested locally to be working on alpine linux, so good enough for now. - #- julia: 1.0 - # os: linux - # env: BINARYPROVIDER_DOWNLOAD_ENGINE="busybox wget" BINARYPROVIDER_COMPRESSION_ENGINE="busybox tar" - # Ironic. He could provide binaries for others but not himself... addons: apt: diff --git a/Project.toml b/Project.toml index bab1844..9f19fbc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,18 +1,18 @@ name = "BinaryProvider" uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" -version = "0.5.5" +version = "0.6.0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" [compat] -julia = "0.7, 1.0" +julia = "1.3" [extras] -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Pkg", "Test"] +test = ["Test"] diff --git a/appveyor.yml b/appveyor.yml index ef1ff1a..99b8b38 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ environment: matrix: - - julia_version: 1.0 + - julia_version: 1.3 - julia_version: latest platform: diff --git a/src/BinaryProvider.jl b/src/BinaryProvider.jl index 79386bd..6c4dcd1 100644 --- a/src/BinaryProvider.jl +++ b/src/BinaryProvider.jl @@ -1,23 +1,34 @@ module BinaryProvider using Libdl, Logging +using Pkg, Pkg.PlatformEngines, Pkg.BinaryPlatforms +import Pkg.PlatformEngines: package, download +import Pkg.BinaryPlatforms: Linux +export platform_key, platform_key_abi, platform_dlext, valid_dl_path, + triplet, select_platform, platforms_match, + Linux, MacOS, Windows, FreeBSD, + parse_7z_list, parse_tar_list, verify, + download_verify, unpack, package, download_verify_unpack, + list_tarball_files, list_tarball_symlinks + +# Some compatibility mapping +const choose_download = Pkg.BinaryPlatforms.select_platform +Linux(arch::Symbol, libc::Symbol) = Linux(arch; libc=libc) +function platform_key(machine::AbstractString = Sys.MACHINE) + Base.depwarn("platform_key() is deprecated, use platform_key_abi() from now on", :binaryprovider_platform_key) + platkey = platform_key_abi(machine) + return typeof(platkey)(arch(platkey); libc=libc(platkey), call_abi=call_abi(platkey)) +end # Utilities for controlling verbosity include("LoggingUtils.jl") # Include our subprocess running functionality include("OutputCollector.jl") -# External utilities such as downloading/decompressing tarballs -include("PlatformEngines.jl") -# Platform naming -include("PlatformNames.jl") # Everything related to file/path management include("Prefix.jl") # Abstraction of "needing" a file, that would trigger an install include("Products.jl") -# Compat shims -include("CompatShims.jl") - function __init__() global global_prefix diff --git a/src/CompatShims.jl b/src/CompatShims.jl deleted file mode 100644 index 534d38d..0000000 --- a/src/CompatShims.jl +++ /dev/null @@ -1,20 +0,0 @@ -# Compatibility shims for old BP versions, when we make a change - -for T in [:Linux, :MacOS, :Windows, :FreeBSD] - @eval $T(arch, libc) = $T(arch; libc=libc) - @eval $T(arch, libc, call_abi) = $T(arch; libc=libc, call_abi=call_abi) -end - -# Compatibility shim to deal with old build.jl files that don't yet use download_info() -function platform_key(machine::AbstractString = Sys.MACHINE) - Base.depwarn("platform_key() is deprecated, use platform_key_abi() from now on", :binaryprovider_platform_key) - platkey = platform_key_abi(machine) - return typeof(platkey)(arch(platkey), libc(platkey), call_abi(platkey)) -end - - -# TODO: fill in better upper bound here when #27674 is backported -# ref: https://github.com/JuliaLang/julia/pull/27674 -if v"0.7.0" <= VERSION < v"0.7.1" - Base.peek(io::Base.AbstractPipe) = Base.peek(Base.pipe_reader(io)) -end diff --git a/src/PlatformEngines.jl b/src/PlatformEngines.jl deleted file mode 100644 index 78f75df..0000000 --- a/src/PlatformEngines.jl +++ /dev/null @@ -1,906 +0,0 @@ -# In this file, we setup the `gen_download_cmd()`, `gen_unpack_cmd()` and -# `gen_package_cmd()` functions by providing methods to probe the environment -# and determine the most appropriate platform binaries to call. - -export gen_download_cmd, gen_unpack_cmd, gen_package_cmd, gen_list_tarball_cmd, - parse_tarball_listing, gen_sh_cmd, parse_7z_list, parse_tar_list, - download_verify_unpack, download_verify, unpack - -""" - gen_download_cmd(url::AbstractString, out_path::AbstractString) - -Return a `Cmd` that will download resource located at `url` and store it at -the location given by `out_path`. - -This method is initialized by `probe_platform_engines()`, which should be -automatically called upon first import of `BinaryProvider`. -""" -gen_download_cmd = (url::AbstractString, out_path::AbstractString) -> - error("Call `probe_platform_engines()` before `gen_download_cmd()`") - -""" - gen_unpack_cmd(tarball_path::AbstractString, out_path::AbstractString; excludelist::Union{AbstractString, Nothing} = nothing) - -Return a `Cmd` that will unpack the given `tarball_path` into the given -`out_path`. If `out_path` is not already a directory, it will be created. -excludlist is an optional file which contains a list of files that is not unpacked -This option is mainyl used to exclude symlinks from extraction (see: `copyderef`) - -This method is initialized by `probe_platform_engines()`, which should be -automatically called upon first import of `BinaryProvider`. -""" -gen_unpack_cmd = (tarball_path::AbstractString, out_path::AbstractString; excludelist::Union{AbstractString, Nothing} = nothing) -> - error("Call `probe_platform_engines()` before `gen_unpack_cmd()`") - -""" - gen_package_cmd(in_path::AbstractString, tarball_path::AbstractString) - -Return a `Cmd` that will package up the given `in_path` directory into a -tarball located at `tarball_path`. - -This method is initialized by `probe_platform_engines()`, which should be -automatically called upon first import of `BinaryProvider`. -""" -gen_package_cmd = (in_path::AbstractString, tarball_path::AbstractString) -> - error("Call `probe_platform_engines()` before `gen_package_cmd()`") - -""" - gen_list_tarball_cmd(tarball_path::AbstractString) - -Return a `Cmd` that will list the files contained within the tarball located at -`tarball_path`. The list will not include directories contained within the -tarball. - -This method is initialized by `probe_platform_engines()`, which should be -automatically called upon first import of `BinaryProvider`. -""" -gen_list_tarball_cmd = (tarball_path::AbstractString) -> - error("Call `probe_platform_engines()` before `gen_list_tarball_cmd()`") - -""" - parse_tarball_listing(output::AbstractString) - -Parses the result of `gen_list_tarball_cmd()` into something useful. - -This method is initialized by `probe_platform_engines()`, which should be -automatically called upon first import of `BinaryProvider`. -""" -parse_tarball_listing = (output::AbstractString) -> - error("Call `probe_platform_engines()` before `parse_tarball_listing()`") - -""" - gen_sh_cmd(cmd::Cmd) - -Runs a command using `sh`. On Unices, this will default to the first `sh` -found on the `PATH`, however on Windows if that is not found it will fall back -to the `sh` provided by the `busybox.exe` shipped with Julia. - -This method is initialized by `probe_platform_engines()`, which should be -automatically called upon first import of `BinaryProvider`. -""" -gen_sh_cmd = (cmd::Cmd) -> - error("Call `probe_platform_engines()` before `gen_sh_cmd()`") - - -""" - probe_cmd(cmd::Cmd; verbose::Bool = false) - -Returns `true` if the given command executes successfully, `false` otherwise. -""" -function probe_cmd(cmd::Cmd; verbose::Bool = false) - if verbose - @info("Probing $(cmd.exec[1]) as a possibility...") - end - try - success(cmd) - if verbose - @info(" Probe successful for $(cmd.exec[1])") - end - return true - catch - return false - end -end - -""" - probe_symlink_creation(dest::AbstractString) - -Probes whether we can create a symlink within the given destination directory, -to determine whether a particular filesystem is "symlink-unfriendly". -""" -function probe_symlink_creation(dest::AbstractString) - while !isdir(dest) - dest = dirname(dest) - end - - # Build arbitrary (non-existent) file path name - link_path = joinpath(dest, "binaryprovider_symlink_test") - while ispath(link_path) - link_path *= "1" - end - - loglevel = Logging.min_enabled_level(current_logger()) - try - disable_logging(Logging.Warn) - symlink("foo", link_path) - return true - catch e - if isa(e, Base.IOError) - return false - end - rethrow(e) - finally - disable_logging(loglevel-1) - rm(link_path; force=true) - end -end - -""" - probe_platform_engines!(;verbose::Bool = false) - -Searches the environment for various tools needed to download, unpack, and -package up binaries. Searches for a download engine to be used by -`gen_download_cmd()` and a compression engine to be used by `gen_unpack_cmd()`, -`gen_package_cmd()`, `gen_list_tarball_cmd()` and `parse_tarball_listing()`, as -well as a `sh` execution engine for `gen_sh_cmd()`. Running this function -will set the global functions to their appropriate implementations given the -environment this package is running on. - -This probing function will automatically search for download engines using a -particular ordering; if you wish to override this ordering and use one over all -others, set the `BINARYPROVIDER_DOWNLOAD_ENGINE` environment variable to its -name, and it will be the only engine searched for. For example, put: - - ENV["BINARYPROVIDER_DOWNLOAD_ENGINE"] = "fetch" - -within your `~/.juliarc.jl` file to force `fetch` to be used over `curl`. If -the given override does not match any of the download engines known to this -function, a warning will be printed and the typical ordering will be performed. - -Similarly, if you wish to override the compression engine used, set the -`BINARYPROVIDER_COMPRESSION_ENGINE` environment variable to its name (e.g. `7z` -or `tar`) and it will be the only engine searched for. If the given override -does not match any of the compression engines known to this function, a warning -will be printed and the typical searching will be performed. - -If `verbose` is `true`, print out the various engines as they are searched. -""" -function probe_platform_engines!(;verbose::Bool = false) - global gen_download_cmd, gen_list_tarball_cmd, gen_package_cmd - global gen_unpack_cmd, parse_tarball_listing, gen_sh_cmd - global gen_symlink_parser - - agent = "BinaryProvider.jl (https://github.com/JuliaPackaging/BinaryProvider.jl)" - # download_engines is a list of (test_cmd, download_opts_functor) - # The probulator will check each of them by attempting to run `$test_cmd`, - # and if that works, will set the global download functions appropriately. - download_engines = [ - (`curl --help`, (url, path) -> `curl -H "User-Agent: $agent" -C - -\# -f -o $path -L $url`), - (`wget --help`, (url, path) -> `wget --tries=5 -U $agent -c -O $path $url`), - (`fetch --help`, (url, path) -> `fetch --user-agent=$agent -f $path $url`), - (`busybox wget --help`, (url, path) -> `busybox wget -U $agent -c -O $path $url`), - ] - Sys.isapple() && pushfirst!(download_engines, (`/usr/bin/curl --help`, (url, path) -> `/usr/bin/curl -H "User-Agent: $agent" -C - -\# -f -o $path -L $url`)) - - # 7z is rather intensely verbose. We also want to try running not only - # `7z` but also a direct path to the `7z.exe` bundled with Julia on - # windows, so we create generator functions to spit back functors to invoke - # the correct 7z given the path to the executable: - unpack_7z = (exe7z) -> begin - return (tarball_path, out_path, excludelist = nothing) -> - pipeline(pipeline(`$exe7z x $(tarball_path) -y -so`, - `$exe7z x -si -y -ttar -o$(out_path) $(excludelist == nothing ? [] : "-x@$(excludelist)")`), stdout = devnull, stderr = devnull) - end - package_7z = (exe7z) -> begin - return (in_path, tarball_path) -> - pipeline(pipeline(`$exe7z a -ttar -so a.tar "$(joinpath(".",in_path,"*"))"`, - `$exe7z a -si $(tarball_path)`), stdout = devnull, stderr = devnull) - end - list_7z = (exe7z) -> begin - return (path; verbose = false) -> - pipeline(`$exe7z x $path -so`, `$exe7z l -ttar -y -si $(verbose ? ["-slt"] : [])`) - end - - # Tar is rather less verbose, and we don't need to search multiple places - # for it, so just rely on PATH to have `tar` available for us: - - # compression_engines is a list of (test_cmd, unpack_opts_functor, - # package_opts_functor, list_opts_functor, parse_functor). The probulator - # will check each of them by attempting to run `$test_cmd`, and if that - # works, will set the global compression functions appropriately. - - # the regex at the last position is meant for parsing the symlinks from verbose 7z-listing - # "Path = ([^\r\n]+)\r?\n" matches the symlink name which is followed by an optional return and a new line - # (?:[^\r\n]+\r?\n)+ = a group of non-empty lines (information belonging to one file is written as a block of lines followed by an empty line) - # more info on regex and a powerful online tester can be found at https://regex101.com - # Symbolic Link = ([^\r\n]+)"s) matches the source filename - # Demo 7z listing of tar files: - # 7-Zip [64] 16.04 : Copyright (c) 1999-2016 Igor Pavlov : 2016-10-04 - # - # - # Listing archive: - # -- - # Path = - # Type = tar - # Code Page = UTF-8 - # - # ---------- - # Path = . - # Folder = + - # Size = 0 - # Packed Size = 0 - # Modified = 2018-08-22 11:44:23 - # Mode = 0rwxrwxr-x - # User = travis - # Group = travis - # Symbolic Link = - # Hard Link = - - # Path = .\lib\libpng.a - # Folder = - - # Size = 10 - # Packed Size = 0 - # Modified = 2018-08-22 11:44:51 - # Mode = 0rwxrwxrwx - # User = travis - # Group = travis - # Symbolic Link = libpng16.a - # Hard Link = - # - # Path = .\lib\libpng16.a - # Folder = - - # Size = 334498 - # Packed Size = 334848 - # Modified = 2018-08-22 11:44:49 - # Mode = 0rw-r--r-- - # User = travis - # Group = travis - # Symbolic Link = - # Hard Link = - gen_7z = (p) -> (unpack_7z(p), package_7z(p), list_7z(p), parse_7z_list, r"Path = ([^\r\n]+)\r?\n(?:[^\r\n]+\r?\n)+Symbolic Link = ([^\r\n]+)"s) - compression_engines = Tuple[] - - (tmpfile, io) = mktemp() - write(io, "Demo file for tar listing (Julia package BinaryProvider.jl)") - close(io) - - for tar_cmd in [`tar`, `busybox tar`] - # try to determine the tar list format - local symlink_parser - try - # Windows 10 now has a `tar` but it needs the `-f -` flag to use stdin/stdout - # The Windows 10 tar does not work on substituted drives (`subst U: C:\Users`) - # If a drive letter is part of the filename, then tar spits out a warning on stderr: - # "tar: Removing leading drive letter from member names" - # Therefore we cd to tmpdir() first - cd(tempdir()) do - tarListing = read(pipeline(`$tar_cmd -cf - $(basename(tmpfile))`, `$tar_cmd -tvf -`), String) - end - # obtain the text of the line before the filename - m = match(Regex("((?:\\S+\\s+)+?)$tmpfile"), tarListing)[1] - # count the number of words before the filename - nargs = length(split(m, " "; keepempty = false)) - # build a regex for catching the symlink: - # "^l" = line starting with l - # "(?:\S+\s+){$nargs} = nargs non-capturing groups of many non-spaces "\S+" and many spaces "\s+" - # "(.+?)" = a non-greedy sequence of characters: the symlink - # "(?: -> (.+?))?" = an optional group of " -> " followed by a non-greedy sequence of characters: the source of the link - # "\r?\$" = matches the end of line with an optional return character for some OSes - # Demo listings - # drwxrwxr-x 0 sabae sabae 0 Sep 5 2018 collapse_the_symlink/ - # lrwxrwxrwx 0 sabae sabae 0 Sep 5 2018 collapse_the_symlink/foo -> foo.1 - # -rw-rw-r-- 0 sabae sabae 0 Sep 5 2018 collapse_the_symlink/foo.1 - # lrwxrwxrwx 0 sabae sabae 0 Sep 5 2018 collapse_the_symlink/foo.1.1 -> foo.1 - # lrwxrwxrwx 0 sabae sabae 0 Sep 5 2018 collapse_the_symlink/broken -> obviously_broken - # - # drwxrwxr-x sabae/sabae 0 2018-09-05 18:19 collapse_the_symlink/ - # lrwxrwxrwx sabae/sabae 0 2018-09-05 18:19 collapse_the_symlink/foo -> foo.1 - # - # lrwxrwxr-x 1000/1000 498007696 2009-11-27 00:14:00 link1 -> source1 - # lrw-rw-r-- 1000/1000 1359020032 2019-06-03 12:02:03 link2 -> sourcedir/source2 - # - # now a pathological link "2009 link with blanks" - # this can only be tracked by determining the tar format beforehand: - # lrw-rw-r-- 0 1000 1000 1359020032 Jul 8 2009 2009 link with blanks -> target with blanks - symlink_parser = Regex("^l(?:\\S+\\s+){$nargs}(.+?)(?: -> (.+?))?\\r?\$", "m") - catch - # generic expression for symlink parsing - # this will fail, if the symlink contains space characters (which is highly improbable, though) - # "^l.+?" = a line starting with an "l" followed by a sequence of non-greedy characters - # \S+? the filename consisting of non-space characters, the rest as above - symlink_parser = r"^l.+? (\S+?)(?: -> (.+?))?\r?$"m - end - # Some tar's aren't smart enough to auto-guess decompression method. :( - unpack_tar = (tarball_path, out_path, excludelist = nothing) -> begin - Jjz = "z" - if endswith(tarball_path, ".xz") - Jjz = "J" - elseif endswith(tarball_path, ".bz2") - Jjz = "j" - end - return `$tar_cmd -x$(Jjz)f $(tarball_path) -C$(out_path) $(excludelist == nothing ? [] : "-X$(excludelist)")` - end - package_tar = (in_path, tarball_path) -> begin - Jjz = "z" - if endswith(tarball_path, ".xz") - Jjz = "J" - elseif endswith(tarball_path, ".bz2") - Jjz = "j" - end - return `$tar_cmd -c$(Jjz)vf $tarball_path -C$(in_path) .` - end - list_tar = (in_path; verbose = false) -> begin - Jjz = "z" - if endswith(in_path, ".xz") - Jjz = "J" - elseif endswith(in_path, ".bz2") - Jjz = "j" - end - return `$tar_cmd $(verbose ? "-t$(Jjz)vf" : "-t$(Jjz)f") $in_path` - end - push!(compression_engines, ( - `$tar_cmd --help`, - unpack_tar, - package_tar, - list_tar, - parse_tar_list, - symlink_parser - )) - end - rm(tmpfile, force = true) - - # sh_engines is just a list of Cmds-as-paths - sh_engines = [ - `sh`, - ] - - # For windows, we need to tweak a few things, as the tools available differ - @static if Sys.iswindows() - # For download engines, we will most likely want to use powershell. - # Let's generate a functor to return the necessary powershell magics - # to download a file, given a path to the powershell executable - psh_download = (psh_path) -> begin - return (url, path) -> begin - webclient_code = """ - [System.Net.ServicePointManager]::SecurityProtocol = - [System.Net.SecurityProtocolType]::Tls12; - \$webclient = (New-Object System.Net.Webclient); - \$webclient.UseDefaultCredentials = \$true; - \$webclient.Proxy.Credentials = \$webclient.Credentials; - \$webclient.Headers.Add("user-agent", "$agent"); - \$webclient.DownloadFile("$url", "$path") - """ - replace(webclient_code, "\n" => " ") - return `$psh_path -NoProfile -Command "$webclient_code"` - end - end - - # We want to search both the `PATH`, and the direct path for powershell - psh_path = joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\WindowsPowerShell\\v1.0\\powershell.exe") - prepend!(download_engines, [ - (`$psh_path -Command ""`, psh_download(psh_path)) - ]) - prepend!(download_engines, [ - (`powershell -Command ""`, psh_download(`powershell`)) - ]) - - # We greatly prefer `7z` as a compression engine on Windows - prepend!(compression_engines, [(`7z --help`, gen_7z("7z")...)]) - - # On windows, we bundle 7z with Julia, so try invoking that directly - exe7z = joinpath(Sys.BINDIR, "7z.exe") - prepend!(compression_engines, [(`$exe7z --help`, gen_7z(exe7z)...)]) - - # And finally, we want to look for sh as busybox as well: - busybox = joinpath(Sys.BINDIR, "busybox.exe") - prepend!(sh_engines, [(`$busybox sh`)]) - end - - # Allow environment override - if haskey(ENV, "BINARYPROVIDER_DOWNLOAD_ENGINE") - engine = ENV["BINARYPROVIDER_DOWNLOAD_ENGINE"] - es = split(engine) - dl_ngs = filter(e -> e[1].exec[1:length(es)] == es, download_engines) - if isempty(dl_ngs) - all_ngs = join([d[1].exec[1] for d in download_engines], ", ") - warn_msg = "Ignoring BINARYPROVIDER_DOWNLOAD_ENGINE as its value " - warn_msg *= "of `$(engine)` doesn't match any known valid engines." - warn_msg *= " Try one of `$(all_ngs)`." - @warn(warn_msg) - else - # If BINARYPROVIDER_DOWNLOAD_ENGINE matches one of our download engines, - # then restrict ourselves to looking only at that engine - download_engines = dl_ngs - end - end - - if haskey(ENV, "BINARYPROVIDER_COMPRESSION_ENGINE") - engine = ENV["BINARYPROVIDER_COMPRESSION_ENGINE"] - es = split(engine) - comp_ngs = filter(e -> e[1].exec[1:length(es)] == es, compression_engines) - if isempty(comp_ngs) - all_ngs = join([c[1].exec[1] for c in compression_engines], ", ") - warn_msg = "Ignoring BINARYPROVIDER_COMPRESSION_ENGINE as its " - warn_msg *= "value of `$(engine)` doesn't match any known valid " - warn_msg *= "engines. Try one of `$(all_ngs)`." - @warn(warn_msg) - else - # If BINARYPROVIDER_COMPRESSION_ENGINE matches one of our download - # engines, then restrict ourselves to looking only at that engine - compression_engines = comp_ngs - end - end - - download_found = false - compression_found = false - sh_found = false - - if verbose - @info("Probing for download engine...") - end - - # Search for a download engine - for (test, dl_func) in download_engines - if probe_cmd(`$test`; verbose=verbose) - # Set our download command generator - gen_download_cmd = dl_func - download_found = true - - if verbose - @info("Found download engine $(test.exec[1])") - end - break - end - end - - if verbose - @info("Probing for compression engine...") - end - - # Search for a compression engine - for (test, unpack, package, list, parse, parse_symlinks) in compression_engines - if probe_cmd(`$test`; verbose=verbose) - # Set our compression command generators - gen_unpack_cmd = unpack - gen_package_cmd = package - gen_list_tarball_cmd = list - parse_tarball_listing = parse - gen_symlink_parser = parse_symlinks - - if verbose - @info("Found compression engine $(test.exec[1])") - end - - compression_found = true - break - end - end - - if verbose - @info("Probing for sh engine...") - end - - for path in sh_engines - if probe_cmd(`$path --help`; verbose=verbose) - gen_sh_cmd = (cmd) -> `$path -c $cmd` - if verbose - @info("Found sh engine $(path.exec[1])") - end - sh_found = true - break - end - end - - - # Build informative error messages in case things go sideways - errmsg = "" - if !download_found - errmsg *= "No download engines found. We looked for: " - errmsg *= join([d[1].exec[1] for d in download_engines], ", ") - errmsg *= ". Install one and ensure it is available on the path.\n" - end - - if !compression_found - errmsg *= "No compression engines found. We looked for: " - errmsg *= join([c[1].exec[1] for c in compression_engines], ", ") - errmsg *= ". Install one and ensure it is available on the path.\n" - end - - if !sh_found && verbose - @warn("No sh engines found. Test suite will fail.") - end - - # Error out if we couldn't find something - if !download_found || !compression_found - error(errmsg) - end -end - -""" - parse_7z_list(output::AbstractString) - -Given the output of `7z l`, parse out the listed filenames. This funciton used -by `list_tarball_files`. -""" -function parse_7z_list(output::AbstractString) - lines = [chomp(l) for l in split(output, "\n")] - - # If we didn't get anything, complain immediately - if isempty(lines) - return [] - end - - # Remove extraneous "\r" for windows platforms - for idx in 1:length(lines) - if endswith(lines[idx], '\r') - lines[idx] = lines[idx][1:end-1] - end - end - - # Find index of " Name". Have to `collect()` as `findfirst()` doesn't work with - # generators: https://github.com/JuliaLang/julia/issues/16884 - header_row = findfirst(collect(occursin(" Name", l) && occursin(" Attr", l) for l in lines)) - name_idx = findfirst("Name", lines[header_row])[1] - attr_idx = findfirst("Attr", lines[header_row])[1] - 1 - - # Filter out only the names of files, ignoring directories - lines = [l[name_idx:end] for l in lines if length(l) > name_idx && l[attr_idx] != 'D'] - if isempty(lines) - return [] - end - - # Extract within the bounding lines of ------------ - bounds = [i for i in 1:length(lines) if all([c for c in lines[i]] .== Ref('-'))] - lines = lines[bounds[1]+1:bounds[2]-1] - - # Eliminate `./` prefix, if it exists - for idx in 1:length(lines) - if startswith(lines[idx], "./") || startswith(lines[idx], ".\\") - lines[idx] = lines[idx][3:end] - end - end - - return lines -end - -""" - parse_tar_list(output::AbstractString) - -Given the output of `tar -t`, parse out the listed filenames. This funciton -used by `list_tarball_files`. -""" -function parse_tar_list(output::AbstractString) - lines = [chomp(l) for l in split(output, "\n")] - for idx in 1:length(lines) - if endswith(lines[idx], '\r') - lines[idx] = lines[idx][1:end-1] - end - end - - # Drop empty lines and and directories - lines = [l for l in lines if !isempty(l) && !endswith(l, '/')] - - # Eliminate `./` prefix, if it exists - for idx in 1:length(lines) - if startswith(lines[idx], "./") || startswith(lines[idx], ".\\") - lines[idx] = lines[idx][3:end] - end - end - - # make sure paths are always returned in the system's default way - return Sys.iswindows() ? replace.(lines, ['/' => '\\']) : lines -end - -""" - download(url::AbstractString, dest::AbstractString; - verbose::Bool = false) - -Download file located at `url`, store it at `dest`, continuing if `dest` -already exists and the server and download engine support it. -""" -function download(url::AbstractString, dest::AbstractString; - verbose::Bool = false) - download_cmd = gen_download_cmd(url, dest) - if verbose - @info("Downloading $(url) to $(dest)...") - end - oc = OutputCollector(download_cmd; verbose=verbose) - try - if !wait(oc) - error() - end - catch e - if isa(e, InterruptException) - rethrow() - end - error("Could not download $(url) to $(dest):\n$(e)") - end -end - -""" - download_verify(url::AbstractString, hash::AbstractString, - dest::AbstractString; verbose::Bool = false, - force::Bool = false, quiet_download::Bool = false) - -Download file located at `url`, verify it matches the given `hash`, and throw -an error if anything goes wrong. If `dest` already exists, just verify it. If -`force` is set to `true`, overwrite the given file if it exists but does not -match the given `hash`. - -This method returns `true` if the file was downloaded successfully, `false` -if an existing file was removed due to the use of `force`, and throws an error -if `force` is not set and the already-existent file fails verification, or if -`force` is set, verification fails, and then verification fails again after -redownloading the file. - -If `quiet_download` is set to `false` (the default), this method will print to -stdout when downloading a new file. If it is set to `true` (and `verbose` is -set to `false`) the downloading process will be completely silent. If -`verbose` is set to `true`, messages about integrity verification will be -printed in addition to messages regarding downloading. -""" -function download_verify(url::AbstractString, hash::AbstractString, - dest::AbstractString; verbose::Bool = false, - force::Bool = false, quiet_download::Bool = false) - # Whether the file existed in the first place - file_existed = false - - if isfile(dest) - file_existed = true - if verbose - info_onchange( - "Destination file $(dest) already exists, verifying...", - "download_verify_$(dest)", - @__LINE__, - ) - end - - # verify download, if it passes, return happy. If it fails, (and - # `force` is `true`, re-download!) - try - verify(dest, hash; verbose=verbose) - return true - catch e - if isa(e, InterruptException) - rethrow() - end - if !force - rethrow() - end - if verbose - info_onchange( - "Verification failed, re-downloading...", - "download_verify_$(dest)", - @__LINE__, - ) - end - end - end - - # Make sure the containing folder exists - mkpath(dirname(dest)) - - try - # Download the file, optionally continuing - download(url, dest; verbose=verbose || !quiet_download) - - verify(dest, hash; verbose=verbose) - catch e - if isa(e, InterruptException) - rethrow() - end - # If the file already existed, it's possible the initially downloaded chunk - # was bad. If verification fails after downloading, auto-delete the file - # and start over from scratch. - if file_existed - if verbose - @info("Continued download didn't work, restarting from scratch") - end - rm(dest; force=true) - - # Download and verify from scratch - download(url, dest; verbose=verbose || !quiet_download) - verify(dest, hash; verbose=verbose) - else - # If it didn't verify properly and we didn't resume, something is - # very wrong and we must complain mightily. - rethrow() - end - end - - # If the file previously existed, this means we removed it (due to `force`) - # and redownloaded, so return `false`. If it didn't exist, then this means - # that we successfully downloaded it, so return `true`. - return !file_existed -end - -""" - package(src_dir::AbstractString, tarball_path::AbstractString; - verbose::Bool = false) - -Compress `src_dir` into a tarball located at `tarball_path`. -""" -function package(src_dir::AbstractString, tarball_path::AbstractString; - verbose::Bool = false) - # For now, use environment variables to set the gzip compression factor to - # level 9, eventually there will be new enough versions of tar everywhere - # to use -I 'gzip -9', or even to switch over to .xz files. - withenv("GZIP" => "-9") do - oc = OutputCollector(gen_package_cmd(src_dir, tarball_path); verbose=verbose) - try - if !wait(oc) - error() - end - catch e - if isa(e, InterruptException) - rethrow() - end - error("Could not package $(src_dir) into $(tarball_path)") - end - end -end - -""" - unpack(tarball_path::AbstractString, dest::AbstractString; - verbose::Bool = false) - -Unpack tarball located at file `tarball_path` into directory `dest`. -""" -function unpack(tarball_path::AbstractString, dest::AbstractString; - verbose::Bool = false) - - # unpack into dest - mkpath(dest) - - # The user can force usage of our dereferencing workarounds for filesystems - # that don't support symlinks, but it is also autodetected. - copyderef = (get(ENV, "BINARYPROVIDER_COPYDEREF", "") == "true") || !probe_symlink_creation(dest) - - # If we should "copyderef" what we do is to unpack everything except symlinks - # then copy the sources of the symlinks to the destination of the symlink instead. - # This is to work around filesystems that are mounted (such as SMBFS filesystems) - # that do not support symlinks. - - excludelist = nothing - - if copyderef - symlinks = list_tarball_symlinks(tarball_path) - if length(symlinks) > 0 - (excludelist, io) = mktemp() - write(io, join([s[1] for s in symlinks], "\n")) - close(io) - end - end - - oc = OutputCollector(gen_unpack_cmd(tarball_path, dest, excludelist); verbose=verbose) - try - if !wait(oc) - error() - end - catch e - if isa(e, InterruptException) - rethrow() - end - error("Could not unpack $(tarball_path) into $(dest)") - end - - if copyderef && length(symlinks) > 0 - @info("Replacing symlinks in tarball by their source files ...\n" * join(string.(symlinks),"\n")) - for s in symlinks - sourcefile = normpath(joinpath(dest, s[2])) - destfile = normpath(joinpath(dest, s[1])) - - if isfile(sourcefile) - cp(sourcefile, destfile, force = true) - else - @warn("Symlink source '$sourcefile' does not exist!") - end - end - rm(excludelist; force = true) - end -end - - -""" - download_verify_unpack(url::AbstractString, hash::AbstractString, - dest::AbstractString; tarball_path = nothing, - verbose::Bool = false, ignore_existence::Bool = false, - force::Bool = false) - -Helper method to download tarball located at `url`, verify it matches the -given `hash`, then unpack it into folder `dest`. In general, the method -`install()` should be used to download and install tarballs into a `Prefix`; -this method should only be used if the extra functionality of `install()` is -undesired. - -If `tarball_path` is specified, the given `url` will be downloaded to -`tarball_path`, and it will not be removed after downloading and verification -is complete. If it is not specified, the tarball will be downloaded to a -temporary location, and removed after verification is complete. - -If `force` is specified, a verification failure will cause `tarball_path` to be -deleted (if it exists), the `dest` folder to be removed (if it exists) and the -tarball to be redownloaded and reverified. If the verification check is failed -a second time, an exception is raised. If `force` is not specified, a -verification failure will result in an immediate raised exception. - -If `ignore_existence` is set, the tarball is unpacked even if the destination -directory already exists. - -Returns `true` if a tarball was actually unpacked, `false` if nothing was -changed in the destination prefix. -""" -function download_verify_unpack(url::AbstractString, - hash::AbstractString, - dest::AbstractString; - tarball_path = nothing, - ignore_existence::Bool = false, - force::Bool = false, - verbose::Bool = false) - # First, determine whether we should keep this tarball around - remove_tarball = false - if tarball_path === nothing - remove_tarball = true - - function url_ext(url) - url = basename(url) - - # Chop off urlparams - qidx = findfirst(isequal('?'), url) - if qidx !== nothing - url = url[1:qidx] - end - - # Try to detect extension - dot_idx = findlast(isequal('.'), url) - if dot_idx === nothing - return nothing - end - - return url[dot_idx+1:end] - end - - # If extension of url contains a recognized extension, use it, otherwise use ".gz" - ext = url_ext(url) - if !(ext in ["tar", "gz", "tgz", "bz2", "xz"]) - ext = "gz" - end - - tarball_path = "$(tempname())-download.$(ext)" - end - - # Download the tarball; if it already existed and we needed to remove it - # then we should remove the unpacked path as well - should_delete = !download_verify(url, hash, tarball_path; - force=force, verbose=verbose) - if should_delete - if verbose - @info("Removing dest directory $(dest) as source tarball changed") - end - rm(dest; recursive=true, force=true) - end - - # If the destination path already exists, don't bother to unpack - if !ignore_existence && isdir(dest) - if verbose - @info("Destination directory $(dest) already exists, returning") - end - - # Signify that we didn't do any unpacking - return false - end - - try - if verbose - @info("Unpacking $(tarball_path) into $(dest)...") - end - unpack(tarball_path, dest; verbose=verbose) - finally - if remove_tarball - rm(tarball_path) - end - end - - # Signify that we did some unpacking! - return true -end diff --git a/src/PlatformNames.jl b/src/PlatformNames.jl deleted file mode 100644 index a36ef69..0000000 --- a/src/PlatformNames.jl +++ /dev/null @@ -1,758 +0,0 @@ -export platform_key, platform_key_abi, platform_dlext, valid_dl_path, arch, libc, call_abi, wordsize, triplet, choose_download, - CompilerABI, Platform, UnknownPlatform, Linux, MacOS, Windows, FreeBSD -import Base: show - -abstract type Platform end - -struct UnknownPlatform <: Platform -end - -# We need to track our compiler ABI compatibility. -struct CompilerABI - # Major GCC version that we're locked into. - # Can be [:gcc4, :gcc5, :gcc6, :gcc7, :gcc8, :gcc_any] - gcc_version::Symbol - - # Whether we're using cxx11abi strings - # Can be [:cxx03, :cxx11, :cxx_any] - cxx_abi::Symbol - - function CompilerABI(gcc_version::Symbol = :gcc_any, cxx_abi::Symbol = :cxx_any) - if !in(gcc_version, [:gcc4, :gcc5, :gcc6, :gcc7, :gcc8, :gcc_any]) - throw(ArgumentError("Unsupported GCC major version '$gcc_version'")) - end - - if !in(cxx_abi, [:cxx03, :cxx11, :cxx_any]) - throw(ArgumentError("Unsupported string ABI '$cxx_abi'")) - end - - return new(gcc_version, cxx_abi) - end -end - -function show(io::IO, cabi::CompilerABI) - write(io, "CompilerABI(") - if cabi.gcc_version != :gcc_any || cabi.cxx_abi != :cxx_any - write(io, "$(repr(cabi.gcc_version))") - end - if cabi.cxx_abi != :cxx_any - write(io, ", $(repr(cabi.cxx_abi))") - end - write(io, ")") -end - - -struct Linux <: Platform - arch::Symbol - libc::Symbol - call_abi::Symbol - compiler_abi::CompilerABI - - function Linux(arch::Symbol; - libc::Symbol=:glibc, - call_abi::Symbol=:default_abi, - compiler_abi::CompilerABI=CompilerABI()) - if !in(arch, [:i686, :x86_64, :aarch64, :powerpc64le, :armv7l]) - throw(ArgumentError("Unsupported architecture '$arch' for Linux")) - end - - # The default libc on Linux is glibc - if libc === :blank_libc - libc = :glibc - end - - if !in(libc, [:glibc, :musl]) - throw(ArgumentError("Unsupported libc '$libc' for Linux")) - end - - # The default calling abi on Linux is blank (e.g. not "eabi" or "eabihf" - # or anything like that, just "blank"), so map over to that by default - # except on armv7l, where we map it over to :eabihf - if call_abi === :default_abi - if arch != :armv7l - call_abi = :blank_abi - else - call_abi = :eabihf - end - end - - if !in(call_abi, [:eabihf, :blank_abi]) - throw(ArgumentError("Unsupported calling abi '$call_abi' for Linux")) - end - - # If we're constructing for armv7l, we MUST have the eabihf abi - if arch == :armv7l && call_abi != :eabihf - throw(ArgumentError("armv7l Linux must use eabihf, not '$call_abi'")) - end - # ...and vice-versa - if arch != :armv7l && call_abi == :eabihf - throw(ArgumentError("eabihf Linux is only on armv7l, not '$arch'!")) - end - - return new(arch, libc, call_abi, compiler_abi) - end -end - -struct MacOS <: Platform - arch::Symbol - libc::Symbol - call_abi::Symbol - compiler_abi::CompilerABI - - # Provide defaults for everything because there's really only one MacOS - # target right now. Maybe someday iOS. :fingers_crossed: - function MacOS(arch::Symbol=:x86_64; - libc::Symbol=:blank_libc, - call_abi::Symbol=:blank_abi, - compiler_abi::CompilerABI=CompilerABI()) - if arch !== :x86_64 - throw(ArgumentError("Unsupported architecture '$arch' for macOS")) - end - if libc !== :blank_libc - throw(ArgumentError("Unsupported libc '$libc' for macOS")) - end - if call_abi !== :blank_abi - throw(ArgumentError("Unsupported abi '$call_abi' for macOS")) - end - - return new(arch, libc, call_abi, compiler_abi) - end -end - -struct Windows <: Platform - arch::Symbol - libc::Symbol - call_abi::Symbol - compiler_abi::CompilerABI - - function Windows(arch::Symbol; - libc::Symbol=:blank_libc, - call_abi::Symbol=:blank_abi, - compiler_abi::CompilerABI=CompilerABI()) - if !in(arch, [:i686, :x86_64]) - throw(ArgumentError("Unsupported architecture '$arch' for Windows")) - end - # We only support the one libc/abi on Windows, so no need to play - # around with "default" values. - if libc !== :blank_libc - throw(ArgumentError("Unsupported libc '$libc' for Windows")) - end - if call_abi !== :blank_abi - throw(ArgumentError("Unsupported abi '$call_abi' for Windows")) - end - - return new(arch, libc, call_abi, compiler_abi) - end -end - -struct FreeBSD <: Platform - arch::Symbol - libc::Symbol - call_abi::Symbol - compiler_abi::CompilerABI - - function FreeBSD(arch::Symbol; - libc::Symbol=:blank_libc, - call_abi::Symbol=:default_abi, - compiler_abi::CompilerABI=CompilerABI()) - # `uname` on FreeBSD reports its architecture as amd64 and i386 instead of x86_64 - # and i686, respectively. In the off chance that Julia hasn't done the mapping for - # us, we'll do it here just in case. - if arch === :amd64 - arch = :x86_64 - elseif arch === :i386 - arch = :i686 - elseif !in(arch, [:i686, :x86_64, :aarch64, :powerpc64le, :armv7l]) - throw(ArgumentError("Unsupported architecture '$arch' for FreeBSD")) - end - - # The only libc we support on FreeBSD is the blank libc, which corresponds to - # FreeBSD's default libc - if libc !== :blank_libc - throw(ArgumentError("Unsupported libc '$libc' for FreeBSD")) - end - - # The default abi on FreeBSD is blank, execpt on armv7l - if call_abi === :default_abi - if arch != :armv7l - call_abi = :blank_abi - else - call_abi = :eabihf - end - end - - if !in(call_abi, [:eabihf, :blank_abi]) - throw(ArgumentError("Unsupported calling abi '$call_abi' for FreeBSD")) - end - - # If we're constructing for armv7l, we MUST have the eabihf abi - if arch == :armv7l && call_abi != :eabihf - throw(ArgumentError("armv7l FreeBSD must use eabihf, no '$call_abi'")) - end - # ...and vice-versa - if arch != :armv7l && call_abi == :eabihf - throw(ArgumentError("eabihf FreeBSD is only on armv7l, not '$arch'!")) - end - - return new(arch, libc, call_abi, compiler_abi) - end -end - -""" - platform_name(p::Platform) - -Get the "platform name" of the given platform. E.g. returns "Linux" for a -`Linux` object, or "Windows" for a `Windows` object. -""" -platform_name(p::Linux) = "Linux" -platform_name(p::MacOS) = "MacOS" -platform_name(p::Windows) = "Windows" -platform_name(p::FreeBSD) = "FreeBSD" -platform_name(p::UnknownPlatform) = "UnknownPlatform" - -""" - arch(p::Platform) - -Get the architecture for the given `Platform` object as a `Symbol`. - -# Examples -```jldoctest -julia> arch(Linux(:aarch64)) -:aarch64 - -julia> arch(MacOS()) -:x86_64 -``` -""" -arch(p::Platform) = p.arch -arch(u::UnknownPlatform) = :unknown - -""" - libc(p::Platform) - -Get the libc for the given `Platform` object as a `Symbol`. - -# Examples -```jldoctest -julia> libc(Linux(:aarch64)) -:glibc - -julia> libc(FreeBSD(:x86_64)) -:default_libc -``` -""" -libc(p::Platform) = p.libc -libc(u::UnknownPlatform) = :unknown - -""" - call_abi(p::Platform) - -Get the calling ABI for the given `Platform` object as a `Symbol`. - -# Examples -```jldoctest -julia> call_abi(Linux(:x86_64)) -:blank_abi - -julia> call_abi(FreeBSD(:armv7l)) -:eabihf -``` -""" -call_abi(p::Platform) = p.call_abi -call_abi(u::UnknownPlatform) = :unknown - -""" - compiler_abi(p::Platform) - -Get the compiler ABI object for the given `Platform` -# Examples -```jldoctest -julia> compiler_abi(Linux(:x86_64)) -CompilerABI(:gcc_any, :cxx_any) - -julia> compiler_abi(Linux(:x86_64; compiler_abi=CompilerABI(:gcc7))) -CompilerABI(:gcc7, :cxx_any) -``` -""" -compiler_abi(p::Platform) = p.compiler_abi -compiler_abi(p::UnknownPlatform) = CompilerABI() - -""" - wordsize(platform) - -Get the word size for the given `Platform` object. - -# Examples -```jldoctest -julia> wordsize(Linux(:arm7vl)) -32 - -julia> wordsize(MacOS()) -64 -``` -""" -wordsize(p::Platform) = (arch(p) === :i686 || arch(p) === :armv7l) ? 32 : 64 -wordsize(u::UnknownPlatform) = 0 - -""" - triplet(platform) - -Get the target triplet for the given `Platform` object as a `String`. - -# Examples -```jldoctest -julia> triplet(MacOS()) -"x86_64-apple-darwin14" - -julia> triplet(Windows(:i686)) -"i686-w64-mingw32" - -julia> triplet(Linux(:armv7l, :default_libc, :default_abi, CompilerABI(:gcc4)) -"arm-linux-gnueabihf-gcc4" -``` -""" -triplet(p::Platform) = string( - arch_str(p), - vendor_str(p), - libc_str(p), - call_abi_str(p), - compiler_abi_str(p), -) -vendor_str(p::Windows) = "-w64-mingw32" -vendor_str(p::MacOS) = "-apple-darwin14" -vendor_str(p::Linux) = "-linux" -vendor_str(p::FreeBSD) = "-unknown-freebsd11.1" - -# Special-case UnknownPlatform -triplet(p::UnknownPlatform) = "unknown-unknown-unknown" - -# Helper functions for Linux and FreeBSD libc/abi mishmashes -arch_str(p::Platform) = (arch(p) == :armv7l) ? "arm" : string(arch(p)) -function libc_str(p::Platform) - if libc(p) == :blank_libc - return "" - elseif libc(p) == :glibc - return "-gnu" - else - return "-$(libc(p))" - end -end -call_abi_str(p::Platform) = (call_abi(p) == :blank_abi) ? "" : string(call_abi(p)) -function compiler_abi_str(cabi::CompilerABI) - str = "" - if cabi.gcc_version != :gcc_any - str *= "-$(cabi.gcc_version)" - end - if cabi.cxx_abi != :cxx_any - str *= "-$(cabi.cxx_abi)" - end - return str -end -compiler_abi_str(p::Platform) = compiler_abi_str(compiler_abi(p)) - -Sys.isapple(p::Platform) = p isa MacOS -Sys.islinux(p::Platform) = p isa Linux -Sys.iswindows(p::Platform) = p isa Windows -Sys.isbsd(p::Platform) = (p isa FreeBSD) || (p isa MacOS) - - -""" - platform_key_abi(machine::AbstractString) - -Returns the platform key for the current platform, or any other though the -the use of the `machine` parameter. -""" -function platform_key_abi(machine::AbstractString) - # We're going to build a mondo regex here to parse everything: - arch_mapping = Dict( - :x86_64 => "(x86_|amd)64", - :i686 => "i\\d86", - :aarch64 => "aarch64", - :armv7l => "arm(v7l)?", - :powerpc64le => "p(ower)?pc64le", - ) - platform_mapping = Dict( - :darwin => "-apple-darwin[\\d\\.]*", - :freebsd => "-(.*-)?freebsd[\\d\\.]*", - :mingw32 => "-w64-mingw32", - :linux => "-(.*-)?linux", - ) - libc_mapping = Dict( - :blank_libc => "", - :glibc => "-gnu", - :musl => "-musl", - ) - call_abi_mapping = Dict( - :blank_abi => "", - :eabihf => "eabihf", - ) - gcc_version_mapping = Dict( - :gcc_any => "", - :gcc4 => "-gcc4", - :gcc5 => "-gcc5", - :gcc6 => "-gcc6", - :gcc7 => "-gcc7", - :gcc8 => "-gcc8", - ) - cxx_abi_mapping = Dict( - :cxx_any => "", - :cxx03 => "-cxx03", - :cxx11 => "-cxx11", - ) - - # Helper function to collapse dictionary of mappings down into a regex of - # named capture groups joined by "|" operators - c(mapping) = string("(",join(["(?<$k>$v)" for (k, v) in mapping], "|"), ")") - - triplet_regex = Regex(string( - "^", - c(arch_mapping), - c(platform_mapping), - c(libc_mapping), - c(call_abi_mapping), - c(gcc_version_mapping), - c(cxx_abi_mapping), - "\$", - )) - - m = match(triplet_regex, machine) - if m != nothing - # Helper function to find the single named field within the giant regex - # that is not `nothing` for each mapping we give it. - get_field(m, mapping) = begin - for k in keys(mapping) - if m[k] != nothing - return k - end - end - end - - # Extract the information we're interested in: - arch = get_field(m, arch_mapping) - platform = get_field(m, platform_mapping) - libc = get_field(m, libc_mapping) - call_abi = get_field(m, call_abi_mapping) - gcc_version = get_field(m, gcc_version_mapping) - cxx_abi = get_field(m, cxx_abi_mapping) - - # First, figure out what platform we're dealing with, then sub that off - # to the appropriate constructor. If a constructor runs into trouble, - # catch the error and return `UnknownPlatform()` here to be nicer to client code. - ctors = Dict(:darwin => MacOS, :mingw32 => Windows, :freebsd => FreeBSD, :linux => Linux) - try - T = ctors[platform] - compiler_abi = CompilerABI(gcc_version, cxx_abi) - return T(arch, libc=libc, call_abi=call_abi, compiler_abi=compiler_abi) - catch - end - end - - @warn("Platform `$(machine)` is not an officially supported platform") - return UnknownPlatform() -end - - -# Define show() for these Platform objects for two reasons: -# - I don't like the `BinaryProvider.` at the beginning of the types; -# it's unnecessary as these are exported -# - I don't like the :blank_*/:any arguments, they're unnecessary -function show(io::IO, p::Platform) - write(io, "$(platform_name(p))($(repr(arch(p)))") - - if libc(p) != :blank_libc - write(io, ", libc=$(repr(libc(p)))") - end - if call_abi(p) != :blank_abi - write(io, ", call_abi=$(repr(call_abi(p)))") - end - cabi = compiler_abi(p) - if cabi.gcc_version != :gcc_any || cabi.cxx_abi != :cxx_any - write(io, ", compiler_abi=$(repr(cabi))") - end - write(io, ")") -end - - -""" - platform_dlext(platform::Platform = platform_key_abi()) - -Return the dynamic library extension for the given platform, defaulting to the -currently running platform. E.g. returns "so" for a Linux-based platform, -"dll" for a Windows-based platform, etc... -""" -platform_dlext(::Linux) = "so" -platform_dlext(::FreeBSD) = "so" -platform_dlext(::MacOS) = "dylib" -platform_dlext(::Windows) = "dll" -platform_dlext(::UnknownPlatform) = "unknown" -platform_dlext() = platform_dlext(platform_key_abi()) - -""" - parse_dl_name_version(path::AbstractString, platform::Platform) - -Given a path to a dynamic library, parse out what information we can -from the filename. E.g. given something like "lib/libfoo.so.3.2", -this function returns `"libfoo", v"3.2"`. If the path name is not a -valid dynamic library, this method throws an error. If no soversion -can be extracted from the filename, as in "libbar.so" this method -returns `"libbar", nothing`. -""" -function parse_dl_name_version(path::AbstractString, platform::Platform) - dlext_regexes = Dict( - # On Linux, libraries look like `libnettle.so.6.3.0` - "so" => r"^(.*?).so((?:\.[\d]+)*)$", - # On OSX, libraries look like `libnettle.6.3.dylib` - "dylib" => r"^(.*?)((?:\.[\d]+)*).dylib$", - # On Windows, libraries look like `libnettle-6.dylib` - "dll" => r"^(.*?)(?:-((?:[\.\d]+)*))?.dll$" - ) - - # Use the regex that matches this platform - dlregex = dlext_regexes[platform_dlext(platform)] - m = match(dlregex, basename(path)) - if m === nothing - throw(ArgumentError("Invalid dynamic library path '$path'")) - end - - # Extract name and version - name = m.captures[1] - version = m.captures[2] - if version === nothing || isempty(version) - version = nothing - else - version = VersionNumber(strip(version, '.')) - end - return name, version -end - -""" - valid_dl_path(path::AbstractString, platform::Platform) - -Return `true` if the given `path` ends in a valid dynamic library filename. -E.g. returns `true` for a path like `"usr/lib/libfoo.so.3.5"`, but returns -`false` for a path like `"libbar.so.f.a"`. -""" -function valid_dl_path(path::AbstractString, platform::Platform) - try - parse_dl_name_version(path, platform) - return true - catch - return false - end -end - -""" - detect_libgfortran_abi(libgfortran_name::AbstractString) - -Examines the given libgfortran SONAME to see what version of GCC corresponds -to the given libgfortran version. -""" -function detect_libgfortran_abi(libgfortran_name::AbstractString, platform::Platform = platform_key_abi(Sys.MACHINE)) - # Extract the version number from this libgfortran. Ironically, parse_dl_name_version() - # wants a Platform, but we may not have initialized the default platform key yet when we - # run this method for the first time (since we need the output of this function to set - # that default platform) so we manually pass in Sys.MACHINE. :P - name, version = parse_dl_name_version(libgfortran_name, platform) - if version === nothing - @warn("Unable to determine libgfortran version from '$(libgfortran_name)'; returning :gcc_any") - return :gcc_any - end - libgfortran_to_gcc = Dict( - 3 => :gcc4, - 4 => :gcc7, - 5 => :gcc8, - ) - if !in(version.major, keys(libgfortran_to_gcc)) - @warn("Unsupported libgfortran version '$version'; returning :gcc_any") - return :gcc_any - end - return libgfortran_to_gcc[version.major] -end - -""" - detect_libgfortran_abi() - -If no parameter is given, introspects the current Julia process to determine -the version of GCC this Julia was built with. -""" -function detect_libgfortran_abi() - libgfortran_paths = filter(x -> occursin("libgfortran", x), dllist()) - if isempty(libgfortran_paths) - # One day, I hope to not be linking against libgfortran in base Julia - return :gcc_any - end - return detect_libgfortran_abi(first(libgfortran_paths)) -end - -""" - detect_libstdcxx_abi() - -Introspects the currently running Julia process to find out what version of libstdc++ -it is linked to (if any), as a proxy for GCC version compatibility. E.g. if we are -linked against libstdc++.so.19, binary dependencies built by GCC 8.1.0 will have linker -errors. This method returns the maximum GCC abi that we can support. -""" -function detect_libstdcxx_abi() - libstdcxx_paths = filter(x -> occursin("libstdc++", x), dllist()) - if isempty(libstdcxx_paths) - # This can happen if we were built by clang, so we don't link against - # libstdc++ at all. - return :gcc_any - end - - # Extract all pieces of `.gnu.version_d` from libstdc++.so, find the `GLIBCXX_*` - # symbols, and use the maximum version of that to find the GLIBCXX ABI version number - #version_symbols = readmeta(first(libstdcxx_paths)) do oh - # unique(vcat((x -> x.names).(ELFVersionData(oh))...)) - #end - #version_symbols = filter(x -> startswith(x, "GLIBCXX_"), version_symbols) - #max_version = maximum([VersionNumber(split(v, "_")[2]) for v in version_symbols]) - - # ^^ Okay, that's really cool, but unfortunately it introduces a dependency on - # ObjectFile which is unacceptable for us. So instead we just brute-force it. - max_version = v"3.4.0" - hdl = dlopen(first(libstdcxx_paths)) - for minor_version in 1:26 - if dlsym_e(hdl, "GLIBCXX_3.4.$(minor_version)") != C_NULL - max_version = VersionNumber("3.4.$(minor_version)") - end - end - dlclose(hdl) - - # Full list available here: https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html - if max_version < v"3.4.18" - @warn "Cannot make sense of autodetected libstdc++ ABI version ('$max_version')" - return :gcc_any - elseif max_version < v"3.4.23" - # If we aren't up to 7.1.0, then we fall all the way back to 4.8.5 - return :gcc4 - elseif max_version < v"3.4.25" - return :gcc7 - else - return :gcc8 - end -end - -""" - detect_cxx11_string_abi() - -Introspects the currently running Julia process to see what version of the C++11 string -ABI it was compiled with. (In reality, it checks for symbols within LLVM, but that is -close enough for our purposes, as you can't mix linkages between Julia and LLVM if they -are not compiled in the same way). -""" -function detect_cxx11_string_abi() - function open_libllvm() - for lib_name in ("libLLVM", "LLVM", "libLLVMSupport") - hdl = dlopen_e(lib_name) - if hdl != C_NULL - return hdl - end - end - error("Unable to open libLLVM!") - end - - hdl = open_libllvm() - # Check for llvm::sys::getProcessTriple(), first without cxx11 tag: - if dlsym_e(hdl, "_ZN4llvm3sys16getProcessTripleEv") != C_NULL - return :cxx03 - elseif dlsym_e(hdl, "_ZN4llvm3sys16getProcessTripleB5cxx11Ev") != C_NULL - return :cxx11 - else - error("Unable to find llvm::sys::getProcessTriple() in libLLVM!") - end -end - -function detect_compiler_abi() - gcc_version = detect_libgfortran_abi() - cxx11_string_abi = detect_cxx11_string_abi() - - # If we have no constraint from libgfortran linkage (impossible within current - # Julia, but let's be planning for the future here) then inspect libstdc++. - if gcc_version == :gcc_any - gcc_version = detect_libstdcxx_abi() - end - - return CompilerABI(gcc_version, cxx11_string_abi) -end - - -# Cache the default platform_key_abi() since that's by far the most common way -# we call platform_key_abi(), and we don't want to parse the same thing over -# and over and over again. Note that we manually slap on a compiler abi -# string onto the end, so as to encode that in Sys.MACHINE like we expect our -# triplets to be encoded. -default_platkey = platform_key_abi(string( - Sys.MACHINE, - compiler_abi_str(detect_compiler_abi()), -)) -function platform_key_abi() - global default_platkey - return default_platkey -end - -function platforms_match(a::Platform, b::Platform) - # Check to see if a and b satisfy the rigid constraints first, these are - # things that are simple equality checks: - function rigid_constraints(a, b) - return (typeof(a) <: typeof(b) || typeof(b) <: typeof(a)) && - (arch(a) == arch(b)) && (libc(a) == libc(b)) && - (call_abi(a) == call_abi(b)) - end - - # The flexible constraints are ones that can do equals, but also have things - # like "any" values, etc.... - function flexible_constraints(a, b) - ac = compiler_abi(a) - bc = compiler_abi(b) - - # Map from GCC version to libgfortran SO version - gfmap = Dict( - :gcc4 => 3, - :gcc5 => 3, - :gcc6 => 3, - :gcc7 => 4, - :gcc8 => 5, - ) - - # We consider two GCC versions to match if their libgfortran - # versions match; e.g. :gcc4 and :gcc5 match. - gcc_match = (ac.gcc_version == :gcc_any - || bc.gcc_version == :gcc_any - || gfmap[ac.gcc_version] == gfmap[bc.gcc_version]) - cxx_match = (ac.cxx_abi == :cxx_any - || bc.cxx_abi == :cxx_any - || ac.cxx_abi == bc.cxx_abi) - return gcc_match && cxx_match - end - - return rigid_constraints(a, b) && flexible_constraints(a, b) -end - -""" - choose_download(download_info::Dict, platform::Platform = platform_key_abi()) - -Given a `download_info` dictionary mapping platforms to some value, choose -the value whose key best matches `platform`, returning `nothing` if no matches -can be found. - -Platform attributes such as architecture, libc, calling ABI, etc... must all -match exactly, however attributes such as compiler ABI can have wildcards -within them such as `:gcc_any` which matches any version of GCC. -""" -function choose_download(download_info::Dict, platform::Platform = platform_key_abi()) - ps = collect(filter(p -> platforms_match(p, platform), keys(download_info))) - - if isempty(ps) - return nothing - end - - # At this point, we may have multiple possibilities. E.g. if, in the future, - # Julia can be built without a direct dependency on libgfortran, we may match - # multiple tarballs that vary only within their libgfortran ABI. To narrow it - # down, we just sort by triplet, then pick the last one. This has the effect - # of generally choosing the latest release (e.g. a `libgfortran5` tarball - # rather than a `libgfortran3` tarball) - p = last(sort(ps, by = p -> triplet(p))) - return download_info[p] -end diff --git a/src/Prefix.jl b/src/Prefix.jl index 2f67ef1..4fb4f17 100644 --- a/src/Prefix.jl +++ b/src/Prefix.jl @@ -7,8 +7,8 @@ using SHA export Prefix, bindir, libdir, includedir, logdir, activate, deactivate, extract_name_version_platform_key, extract_platform_key, isinstalled, install, uninstall, manifest_from_url, manifest_for_file, - list_tarball_files, list_tarball_symlinks, verify, temp_prefix, package - + list_tarball_files, list_tarball_symlinks, verify, temp_prefix, package, + choose_download # Temporary hack around https://github.com/JuliaLang/julia/issues/26685 function safe_isfile(path) @@ -70,10 +70,8 @@ struct Prefix """ Prefix(path::AbstractString) - A `Prefix` represents a binary installation location. There is a default - global `Prefix` (available at `BinaryProvider.global_prefix`) that packages - are installed into by default, however custom prefixes can be created - trivially by simply constructing a `Prefix` with a given `path` to install + A `Prefix` represents a binary installation location. Custom prefixes can + be created by constructing a `Prefix` with a given `path` to install binaries into, likely including folders such as `bin`, `lib`, etc... """ function Prefix(path::AbstractString) @@ -448,165 +446,7 @@ function manifest_for_file(path::AbstractString; error("Could not find $(search_path) in any manifest files") end -""" - list_tarball_files(path::AbstractString; verbose::Bool = false) - -Given a `.tar.gz` filepath, list the compressed contents. -""" -function list_tarball_files(path::AbstractString; verbose::Bool = false) - if !isfile(path) - error("Tarball path $(path) does not exist") - end - - # Run the listing command, then parse the output - oc = OutputCollector(gen_list_tarball_cmd(path); verbose=verbose) - try - if !wait(oc) - error() - end - catch - error("Could not list contents of tarball $(path)") - end - return parse_tarball_listing(collect_stdout(oc)) -end - -""" - list_tarball_symlinks(path::AbstractString; verbose::Bool = false) - -Given a `.tar.gz` filepath, return a dictionary of symlinks in the archive -""" -function list_tarball_symlinks(tarball_path::AbstractString; verbose::Bool = false) - if !isdefined(BinaryProvider, :gen_symlink_parser) - error("Call `probe_platform_engines!()` before `list_tarball_symlinks()`") - end - oc = OutputCollector(gen_list_tarball_cmd(tarball_path; verbose = true); verbose = verbose) - try - if !wait(oc) - error() - end - catch - error("Could not list contents of tarball $(tarball_path)") - end - output = collect_stdout(oc) - - mm = [m.captures for m in eachmatch(gen_symlink_parser, output)] - symlinks = [m[1] => joinpath(dirname(m[1]), m[2]) for m in mm] - return symlinks -end - -""" - verify(path::AbstractString, hash::AbstractString; - verbose::Bool = false, report_cache_status::Bool = false) - -Given a file `path` and a `hash`, calculate the SHA256 of the file and compare -it to `hash`. If an error occurs, `verify()` will throw an error. This method -caches verification results in a `"\$(path).sha256"` file to accelerate re- -verification of files that have been previously verified. If no `".sha256"` -file exists, a full verification will be done and the file will be created, -with the calculated hash being stored within the `".sha256"` file.. If a -`".sha256"` file does exist, its contents are checked to ensure that the hash -contained within matches the given `hash` parameter, and its modification time -shows that the file located at `path` has not been modified since the last -verification. - -If `report_cache_status` is set to `true`, then the return value will be a -`Symbol` giving a granular status report on the state of the hash cache, in -addition to the `true`/`false` signifying whether verification completed -successfully. -""" -function verify(path::AbstractString, hash::AbstractString; verbose::Bool = false, - report_cache_status::Bool = false, hash_path::AbstractString="$(path).sha256") - if length(hash) != 64 - msg = "Hash must be 256 bits (64 characters) long, " - msg *= "given hash is $(length(hash)) characters long" - error(msg) - end - - # First, check to see if the hash cache is consistent - status = :hash_consistent - # First, it must exist - if isfile(hash_path) - # Next, it must contain the same hash as what we're verifying against - if read(hash_path, String) == hash - # Next, it must be no older than the actual path - if stat(hash_path).mtime >= stat(path).mtime - # If all of that is true, then we're good! - if verbose - info_onchange( - "Hash cache is consistent, returning true", - "verify_$(hash_path)", - @__LINE__, - ) - end - status = :hash_cache_consistent - - # If we're reporting our status, then report it! - if report_cache_status - return true, status - else - return true - end - else - if verbose - info_onchange( - "File has been modified, hash cache invalidated", - "verify_$(hash_path)", - @__LINE__, - ) - end - status = :file_modified - end - else - if verbose - info_onchange( - "Verification hash mismatch, hash cache invalidated", - "verify_$(hash_path)", - @__LINE__, - ) - end - status = :hash_cache_mismatch - end - else - if verbose - info_onchange( - "No hash cache found", - "verify_$(hash_path)", - @__LINE__, - ) - end - status = :hash_cache_missing - end - - open(path) do file - calc_hash = bytes2hex(sha256(file)) - if verbose - info_onchange( - "Calculated hash $calc_hash for file $path", - "hash_$(hash_path)", - @__LINE__, - ) - end - - if calc_hash != hash - msg = "Hash Mismatch!\n" - msg *= " Expected sha256: $hash\n" - msg *= " Calculated sha256: $calc_hash" - error(msg) - end - end - - # Save a hash cache if everything worked out fine - open(hash_path, "w") do file - write(file, hash) - end - - if report_cache_status - return true, status - else - return true - end -end """ package(prefix::Prefix, output_base::AbstractString, @@ -647,7 +487,7 @@ function package(prefix::Prefix, end # Package `prefix.path` into the tarball contained at `out_path` - package(prefix.path, out_path; verbose=verbose) + package(prefix.path, out_path) # Also spit out the hash of the archive file hash = open(out_path, "r") do f diff --git a/src/precompile.jl b/src/precompile.jl index 0c03b20..11e3e7f 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -2,62 +2,13 @@ function _precompile_() ccall(:jl_generating_output, Cint, ()) == 1 || return nothing precompile(Tuple{typeof(BinaryProvider.safe_isfile), String}) precompile(Tuple{typeof(BinaryProvider.info_onchange), String, String, Int64}) - precompile(Tuple{typeof(BinaryProvider.detect_libgfortran_abi), String, BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.detect_libgfortran_abi), String, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.parse_7z_list), String}) - precompile(Tuple{typeof(BinaryProvider.platform_key_abi), Base.SubString{String}}) precompile(Tuple{typeof(BinaryProvider.libdir), BinaryProvider.Prefix}) - precompile(Tuple{typeof(BinaryProvider.detect_libgfortran_abi), String, BinaryProvider.Linux}) precompile(Tuple{typeof(BinaryProvider.readuntil_many), Base.Pipe, Array{Char, 1}}) - precompile(Tuple{typeof(BinaryProvider.parse_dl_name_version), String, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.parse_dl_name_version), String, BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.probe_symlink_creation), String}) precompile(Tuple{typeof(BinaryProvider.locate), BinaryProvider.ExecutableProduct}) - precompile(Tuple{typeof(BinaryProvider.extract_platform_key), String}) - precompile(Tuple{typeof(BinaryProvider.triplet), BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.triplet), BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.triplet), BinaryProvider.Linux}) - precompile(Tuple{typeof(BinaryProvider.triplet), BinaryProvider.FreeBSD}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.MacOS, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.platform_key_abi), String}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.Linux, BinaryProvider.Linux}) precompile(Tuple{typeof(BinaryProvider.package), BinaryProvider.Prefix, String, Base.VersionNumber}) - precompile(Tuple{typeof(BinaryProvider.parse_tar_list), String}) - precompile(Tuple{typeof(BinaryProvider.choose_download), Base.Dict{BinaryProvider.Platform, String}, BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.choose_download), Base.Dict{BinaryProvider.Platform, Tuple{String, String}}, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.choose_download), Base.Dict{BinaryProvider.Platform, String}, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.__init__)}) - precompile(Tuple{typeof(BinaryProvider.choose_download), Base.Dict{BinaryProvider.Platform, String}, BinaryProvider.Linux}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.Linux, BinaryProvider.UnknownPlatform}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.Windows, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.wordsize), BinaryProvider.Linux}) - precompile(Tuple{typeof(BinaryProvider.wordsize), BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.wordsize), BinaryProvider.FreeBSD}) - precompile(Tuple{typeof(BinaryProvider.parse_dl_name_version), String, BinaryProvider.Linux}) precompile(Tuple{typeof(BinaryProvider.libdir), BinaryProvider.Prefix, BinaryProvider.Linux}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.UnknownPlatform, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.platform_key_abi)}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.MacOS, BinaryProvider.Linux}) precompile(Tuple{typeof(BinaryProvider.libdir), BinaryProvider.Prefix, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.Linux, BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.list_tarball_files), String}) - precompile(Tuple{typeof(BinaryProvider.arch), BinaryProvider.Linux}) - precompile(Tuple{typeof(BinaryProvider.platforms_match), BinaryProvider.Linux, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.arch), BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.platform_dlext)}) precompile(Tuple{typeof(BinaryProvider.locate), BinaryProvider.LibraryProduct}) - precompile(Tuple{typeof(BinaryProvider.triplet), BinaryProvider.UnknownPlatform}) - precompile(Tuple{typeof(BinaryProvider.platform_dlext), BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.arch), BinaryProvider.FreeBSD}) precompile(Tuple{typeof(BinaryProvider.collect_stdout), BinaryProvider.OutputCollector}) - precompile(Tuple{typeof(BinaryProvider.extract_name_version_platform_key), String}) - precompile(Tuple{typeof(BinaryProvider.platform_dlext), BinaryProvider.FreeBSD}) - precompile(Tuple{typeof(BinaryProvider.platform_dlext), BinaryProvider.Linux}) precompile(Tuple{typeof(BinaryProvider.collect_stderr), BinaryProvider.OutputCollector}) - precompile(Tuple{typeof(BinaryProvider.platform_dlext), BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.platform_dlext), BinaryProvider.UnknownPlatform}) - precompile(Tuple{typeof(BinaryProvider.arch), BinaryProvider.UnknownPlatform}) - precompile(Tuple{typeof(BinaryProvider.valid_dl_path), String, BinaryProvider.Windows}) - precompile(Tuple{typeof(BinaryProvider.valid_dl_path), String, BinaryProvider.MacOS}) - precompile(Tuple{typeof(BinaryProvider.valid_dl_path), String, BinaryProvider.Linux}) end diff --git a/test/LibFoo.jl/deps/build.jl b/test/LibFoo.jl/deps/build.jl index 6dfea3e..090b3a0 100644 --- a/test/LibFoo.jl/deps/build.jl +++ b/test/LibFoo.jl/deps/build.jl @@ -13,16 +13,16 @@ products = Product[ # Download binaries from hosted location bin_prefix = "https://github.com/staticfloat/small_bin/raw/51b13b44feb2a262e2e04690bfa54d03167533f2/libfoo" download_info = Dict( - Linux(:aarch64, :glibc) => ("$bin_prefix/libfoo.aarch64-linux-gnu.tar.gz", "36886ac25cf5678c01fe20630b413f9354b7a3721c6a2c2043162f7ebd147ff5"), - Linux(:armv7l, :glibc) => ("$bin_prefix/libfoo.arm-linux-gnueabihf.tar.gz", "147ebaeb1a722da43ee08705689aed71ac87c3c2c907af047c6721c0025ba383"), - Linux(:powerpc64le, :glibc) => ("$bin_prefix/libfoo.powerpc64le-linux-gnu.tar.gz", "5c35295ac161272ada9a77d1f6b770e30ea864e521e31853258cbc36ad4c4468"), - Linux(:i686, :glibc) => ("$bin_prefix/libfoo.i686-linux-gnu.tar.gz", "97655b6a218d61284723b6923d7c96e6a256fa68b9419d723c588aa24404b102"), - Linux(:x86_64, :glibc) => ("$bin_prefix/libfoo.x86_64-linux-gnu.tar.gz", "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391"), + Linux(:aarch64; libc=:glibc) => ("$bin_prefix/libfoo.aarch64-linux-gnu.tar.gz", "36886ac25cf5678c01fe20630b413f9354b7a3721c6a2c2043162f7ebd147ff5"), + Linux(:armv7l; libc=:glibc) => ("$bin_prefix/libfoo.arm-linux-gnueabihf.tar.gz", "147ebaeb1a722da43ee08705689aed71ac87c3c2c907af047c6721c0025ba383"), + Linux(:powerpc64le; libc=:glibc) => ("$bin_prefix/libfoo.powerpc64le-linux-gnu.tar.gz", "5c35295ac161272ada9a77d1f6b770e30ea864e521e31853258cbc36ad4c4468"), + Linux(:i686; libc=:glibc) => ("$bin_prefix/libfoo.i686-linux-gnu.tar.gz", "97655b6a218d61284723b6923d7c96e6a256fa68b9419d723c588aa24404b102"), + Linux(:x86_64; libc=:glibc) => ("$bin_prefix/libfoo.x86_64-linux-gnu.tar.gz", "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391"), - Linux(:aarch64, :musl) => ("$bin_prefix/libfoo.aarch64-linux-musl.tar.gz", "81751477c1e3ee6c93e1c28ee7db2b99d1eed0d6ce86dc30d64c2e5dd4dfe88d"), - Linux(:armv7l, :musl) => ("$bin_prefix/libfoo.arm-linux-musleabihf.tar.gz", "bb65aad58f2e6fc39dc9688da1bca5e8103a3a3fa67dc589debbd2e98176f0e1"), - Linux(:i686, :musl) => ("$bin_prefix/libfoo.i686-linux-musl.tar.gz", "5f02fd1fe19f3a565fb128d3673b35c7b3214a101cef9dcbb202c0092438a87b"), - Linux(:x86_64, :musl) => ("$bin_prefix/libfoo.x86_64-linux-musl.tar.gz", "ea630600a12d2c1846bc93bcc8d9638a4991f63329205c534d93e0a3de5f641d"), + Linux(:aarch64; libc=:musl) => ("$bin_prefix/libfoo.aarch64-linux-musl.tar.gz", "81751477c1e3ee6c93e1c28ee7db2b99d1eed0d6ce86dc30d64c2e5dd4dfe88d"), + Linux(:armv7l; libc=:musl) => ("$bin_prefix/libfoo.arm-linux-musleabihf.tar.gz", "bb65aad58f2e6fc39dc9688da1bca5e8103a3a3fa67dc589debbd2e98176f0e1"), + Linux(:i686; libc=:musl) => ("$bin_prefix/libfoo.i686-linux-musl.tar.gz", "5f02fd1fe19f3a565fb128d3673b35c7b3214a101cef9dcbb202c0092438a87b"), + Linux(:x86_64; libc=:musl) => ("$bin_prefix/libfoo.x86_64-linux-musl.tar.gz", "ea630600a12d2c1846bc93bcc8d9638a4991f63329205c534d93e0a3de5f641d"), FreeBSD(:x86_64) => ("$bin_prefix/libfoo.x86_64-unknown-freebsd11.1.tar.gz", "5f6edd6247b3685fa5c42c98a53d2a3e1eef6242c2bb3cdbb5fe23f538703fe4"), MacOS(:x86_64) => ("$bin_prefix/libfoo.x86_64-apple-darwin14.tar.gz", "fcc268772d6f21d65b45fcf3854a3142679b78e53c7673dac26c95d6ccc89a24"), @@ -34,7 +34,7 @@ download_info = Dict( if any(!satisfied(p; verbose=verbose) for p in products) try # Download and install binaries - url, tarball_hash = choose_download(download_info) + url, tarball_hash = select_platform(download_info) install(url, tarball_hash; prefix=prefix, force=true, verbose=true) catch e if typeof(e) <: ArgumentError diff --git a/test/runtests.jl b/test/runtests.jl index d6a4a87..9367ca3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,9 +7,6 @@ using SHA # The platform we're running on const platform = platform_key_abi() -# Useful command to launch `sh` on any platform -const sh = gen_sh_cmd - # Output of a few scripts we are going to run const simple_out = "1\n2\n3\n4\n" const long_out = join(["$(idx)\n" for idx in 1:100], "") @@ -19,6 +16,9 @@ const newlines_out = join(["marco$(d)polo$(d)" for d in ("\n","\r","\r\n")], "") # CI debugging easier BinaryProvider.probe_platform_engines!(;verbose=true) +# shell executable for testing +sh = "sh" + # Helper function to strip out color codes from strings to make it easier to # compare output within tests that has been colorized function strip_colorization(s) @@ -33,7 +33,7 @@ end @testset "OutputCollector" begin cd("output_tests") do # Collect the output of `simple.sh`` - oc = OutputCollector(sh(`./simple.sh`)) + oc = OutputCollector(`$sh ./simple.sh`) # Ensure we can wait on it and it exited properly @test wait(oc) @@ -70,7 +70,7 @@ end # Next test a much longer output program cd("output_tests") do - oc = OutputCollector(sh(`./long.sh`)) + oc = OutputCollector(`$sh ./long.sh`) # Test that it worked, we can read it, and tail() works @test wait(oc) @@ -80,7 +80,7 @@ end # Next, test a command that fails cd("output_tests") do - oc = OutputCollector(sh(`./fail.sh`)) + oc = OutputCollector(`$sh ./fail.sh`) @test !wait(oc) @test merge(oc) == "1\n2\n" @@ -89,7 +89,7 @@ end # Next, test a command that kills itself (NOTE: This doesn't work on windows. sigh.) @static if !Sys.iswindows() cd("output_tests") do - oc = OutputCollector(sh(`./kill.sh`)) + oc = OutputCollector(`$sh ./kill.sh`) @test !wait(oc) @test collect_stdout(oc) == "1\n2\n" @@ -98,8 +98,8 @@ end # Next, test reading the output of a pipeline() grepline = pipeline( - sh(`'printf "Hello\nWorld\nJulia\n"'`), - sh(`'while read line; do case $line in *ul*) echo $line; esac; done'`) + `$sh -c 'printf "Hello\nWorld\nJulia\n"'`, + `$sh -c 'while read line; do case $line in *ul*) echo $line; esac; done'` ) oc = OutputCollector(grepline) @@ -108,7 +108,7 @@ end # Next, test that \r and \r\n are treated like \n cd("output_tests") do - oc = OutputCollector(sh(`./newlines.sh`)) + oc = OutputCollector(`$sh ./newlines.sh`) @test wait(oc) @test collect_stdout(oc) == newlines_out @@ -117,7 +117,7 @@ end # Next, test that tee'ing to a stream works cd("output_tests") do ios = IOBuffer() - oc = OutputCollector(sh(`./simple.sh`); tee_stream=ios, verbose=true) + oc = OutputCollector(`$sh ./simple.sh`; tee_stream=ios, verbose=true) @test wait(oc) @test merge(oc) == simple_out @@ -131,7 +131,7 @@ end # Also test that auto-tail'ing can be can be directed to a stream cd("output_tests") do ios = IOBuffer() - oc = OutputCollector(sh(`./fail.sh`); tee_stream=ios) + oc = OutputCollector(`$sh ./fail.sh`; tee_stream=ios) @test !wait(oc) @test merge(oc) == "1\n2\n" @@ -145,7 +145,7 @@ end # Also test that auto-tail'ing can be turned off cd("output_tests") do ios = IOBuffer() - oc = OutputCollector(sh(`./fail.sh`); tee_stream=ios, tail_error=false) + oc = OutputCollector(`$sh ./fail.sh`; tee_stream=ios, tail_error=false) @test !wait(oc) @test merge(oc) == "1\n2\n" @@ -155,187 +155,6 @@ end end end -@testset "PlatformNames" begin - # Ensure the platform type constructors are well behaved - @testset "Platform constructors" begin - @test_throws ArgumentError Linux(:not_a_platform) - @test_throws ArgumentError Linux(:x86_64, :crazy_libc) - @test_throws ArgumentError Linux(:x86_64, :glibc, :crazy_abi) - @test_throws ArgumentError Linux(:x86_64, :glibc, :eabihf) - @test_throws ArgumentError Linux(:armv7l, :glibc, :blank_abi) - @test_throws ArgumentError MacOS(:i686) - @test_throws ArgumentError MacOS(:x86_64, :glibc) - @test_throws ArgumentError MacOS(:x86_64, :blank_libc, :eabihf) - @test_throws ArgumentError Windows(:armv7l) - @test_throws ArgumentError Windows(:x86_64, :glibc) - @test_throws ArgumentError Windows(:x86_64, :blank_libc, :eabihf) - @test_throws ArgumentError FreeBSD(:not_a_platform) - @test_throws ArgumentError FreeBSD(:x86_64, :crazy_libc) - @test_throws ArgumentError FreeBSD(:x86_64, :blank_libc, :crazy_abi) - @test_throws ArgumentError FreeBSD(:x86_64, :blank_libc, :eabihf) - @test_throws ArgumentError FreeBSD(:armv7l, :blank_libc, :blank_abi) - end - - @testset "Platform properties" begin - # Test that we can get that arch of various platforms - @test arch(Linux(:aarch64, :musl)) == :aarch64 - @test arch(Windows(:i686)) == :i686 - @test arch(UnknownPlatform()) == :unknown - @test arch(FreeBSD(:amd64)) == :x86_64 - @test arch(FreeBSD(:i386)) == :i686 - - # Test that our platform_dlext stuff works - @test platform_dlext(Linux(:x86_64)) == platform_dlext(Linux(:i686)) - @test platform_dlext(Windows(:x86_64)) == platform_dlext(Windows(:i686)) - @test platform_dlext(MacOS()) != platform_dlext(Linux(:armv7l)) - @test platform_dlext(FreeBSD(:x86_64)) == platform_dlext(Linux(:x86_64)) - @test platform_dlext(UnknownPlatform()) == "unknown" - @test platform_dlext() == platform_dlext(platform) - - @test wordsize(Linux(:i686)) == wordsize(Linux(:armv7l)) == 32 - @test wordsize(MacOS()) == wordsize(Linux(:aarch64)) == 64 - @test wordsize(FreeBSD(:x86_64)) == wordsize(Linux(:powerpc64le)) == 64 - @test wordsize(UnknownPlatform()) == 0 - - @test triplet(Windows(:i686)) == "i686-w64-mingw32" - @test triplet(Linux(:x86_64, :musl)) == "x86_64-linux-musl" - @test triplet(Linux(:armv7l, :musl)) == "arm-linux-musleabihf" - @test triplet(Linux(:x86_64)) == "x86_64-linux-gnu" - @test triplet(Linux(:armv7l)) == "arm-linux-gnueabihf" - @test triplet(MacOS()) == "x86_64-apple-darwin14" - @test triplet(FreeBSD(:x86_64)) == "x86_64-unknown-freebsd11.1" - @test triplet(FreeBSD(:i686)) == "i686-unknown-freebsd11.1" - @test triplet(UnknownPlatform()) == "unknown-unknown-unknown" - - @test repr(Windows(:x86_64)) == "Windows(:x86_64)" - @test repr(Linux(:x86_64, :glibc, :blank_abi)) == "Linux(:x86_64, libc=:glibc)" - @test repr(MacOS()) == "MacOS(:x86_64)" - @test repr(MacOS(compiler_abi=CompilerABI(:gcc_any, :cxx11))) == "MacOS(:x86_64, compiler_abi=CompilerABI(:gcc_any, :cxx11))" - end - - @testset "Valid DL paths" begin - # Test some valid dynamic library paths - @test valid_dl_path("libfoo.so.1.2.3", Linux(:x86_64)) - @test valid_dl_path("libfoo.1.2.3.so", Linux(:x86_64)) - @test valid_dl_path("libfoo-1.2.3.dll", Windows(:x86_64)) - @test valid_dl_path("libfoo.1.2.3.dylib", MacOS()) - @test !valid_dl_path("libfoo.dylib", Linux(:x86_64)) - @test !valid_dl_path("libfoo.so", Windows(:x86_64)) - @test !valid_dl_path("libfoo.dll", MacOS()) - @test !valid_dl_path("libfoo.so.1.2.3.", Linux(:x86_64)) - @test !valid_dl_path("libfoo.so.1.2a.3", Linux(:x86_64)) - end - - @testset "platform_key_abi parsing" begin - # Make sure the platform_key_abi() with explicit triplet works - @test platform_key_abi("x86_64-linux-gnu") == Linux(:x86_64) - @test platform_key_abi("x86_64-linux-musl") == Linux(:x86_64, libc=:musl) - @test platform_key_abi("i686-unknown-linux-gnu") == Linux(:i686) - @test platform_key_abi("x86_64-apple-darwin14") == MacOS() - @test platform_key_abi("x86_64-apple-darwin17.0.0") == MacOS() - @test platform_key_abi("armv7l-pc-linux-gnueabihf") == Linux(:armv7l) - @test platform_key_abi("armv7l-linux-musleabihf") == Linux(:armv7l, libc=:musl) - @test platform_key_abi("arm-linux-gnueabihf") == Linux(:armv7l) - @test platform_key_abi("aarch64-unknown-linux-gnu") == Linux(:aarch64) - @test platform_key_abi("powerpc64le-linux-gnu") == Linux(:powerpc64le) - @test platform_key_abi("ppc64le-linux-gnu") == Linux(:powerpc64le) - @test platform_key_abi("x86_64-w64-mingw32") == Windows(:x86_64) - @test platform_key_abi("i686-w64-mingw32") == Windows(:i686) - @test platform_key_abi("x86_64-unknown-freebsd11.1") == FreeBSD(:x86_64) - @test platform_key_abi("i686-unknown-freebsd11.1") == FreeBSD(:i686) - @test platform_key_abi("amd64-unknown-freebsd12.0") == FreeBSD(:x86_64) - @test platform_key_abi("i386-unknown-freebsd10.3") == FreeBSD(:i686) - - # Test inclusion of ABI stuff - @test platform_key_abi("x86_64-linux-gnu-gcc7") == Linux(:x86_64, compiler_abi=CompilerABI(:gcc7)) - @test platform_key_abi("x86_64-linux-gnu-gcc4-cxx11") == Linux(:x86_64, compiler_abi=CompilerABI(:gcc4, :cxx11)) - @test platform_key_abi("x86_64-linux-gnu-cxx11") == Linux(:x86_64, compiler_abi=CompilerABI(:gcc_any, :cxx11)) - - # Make sure some of these things are rejected - @test platform_key_abi("totally FREEFORM text!!1!!!1!") == UnknownPlatform() - @test platform_key_abi("invalid-triplet-here") == UnknownPlatform() - @test platform_key_abi("aarch64-linux-gnueabihf") == UnknownPlatform() - @test platform_key_abi("armv7l-linux-gnu") == UnknownPlatform() - @test platform_key_abi("x86_64-w32-mingw64") == UnknownPlatform() - end - - @testset "platforms_match()" begin - # Test our platform matching code - @test BinaryProvider.platforms_match(Linux(:x86_64), Linux(:x86_64)) - @test BinaryProvider.platforms_match(Linux(:x86_64), Linux(:x86_64, compiler_abi=CompilerABI(:gcc7))) - @test BinaryProvider.platforms_match(Linux(:x86_64, compiler_abi=CompilerABI(:gcc7)), Linux(:x86_64)) - - @test !BinaryProvider.platforms_match(Linux(:x86_64), Linux(:i686)) - @test !BinaryProvider.platforms_match(Linux(:x86_64), Windows(:x86_64)) - @test !BinaryProvider.platforms_match(Linux(:x86_64), MacOS()) - @test !BinaryProvider.platforms_match(Linux(:x86_64), UnknownPlatform()) - @test !BinaryProvider.platforms_match(Linux(:x86_64, compiler_abi=CompilerABI(:gcc7)), Linux(:x86_64, compiler_abi=CompilerABI(:gcc8))) - @test !BinaryProvider.platforms_match(Linux(:x86_64, compiler_abi=CompilerABI(:gcc7, :cxx11)), Linux(:x86_64, compiler_abi=CompilerABI(:gcc7, :cxx03))) - - # Test that :gcc4 matches with :gcc5 and :gcc6, as we only care aobut libgfortran version - @test BinaryProvider.platforms_match( - Linux(:x86_64; compiler_abi=CompilerABI(:gcc4)), - Linux(:x86_64; compiler_abi=CompilerABI(:gcc5)), - ) - @test BinaryProvider.platforms_match( - Linux(:x86_64; compiler_abi=CompilerABI(:gcc4)), - Linux(:x86_64; compiler_abi=CompilerABI(:gcc6)), - ) - @test BinaryProvider.platforms_match( - Linux(:x86_64; compiler_abi=CompilerABI(:gcc5)), - Linux(:x86_64; compiler_abi=CompilerABI(:gcc6)), - ) - end - - @testset "DL name/version parsing" begin - # Make sure our version parsing code is working - @test BinaryProvider.parse_dl_name_version("libgfortran.dll", Windows(:x86_64)) == ("libgfortran", nothing) - @test BinaryProvider.parse_dl_name_version("libgfortran-3.dll", Windows(:x86_64)) == ("libgfortran", v"3") - @test BinaryProvider.parse_dl_name_version("libgfortran-3.4.dll", Windows(:x86_64)) == ("libgfortran", v"3.4") - @test BinaryProvider.parse_dl_name_version("libgfortran-3.4a.dll", Windows(:x86_64)) == ("libgfortran-3.4a", nothing) - @test_throws ArgumentError BinaryProvider.parse_dl_name_version("libgfortran", Windows(:x86_64)) - @test BinaryProvider.parse_dl_name_version("libgfortran.dylib", MacOS(:x86_64)) == ("libgfortran", nothing) - @test BinaryProvider.parse_dl_name_version("libgfortran.3.dylib", MacOS(:x86_64)) == ("libgfortran", v"3") - @test BinaryProvider.parse_dl_name_version("libgfortran.3.4.dylib", MacOS(:x86_64)) == ("libgfortran", v"3.4") - @test BinaryProvider.parse_dl_name_version("libgfortran.3.4a.dylib", MacOS(:x86_64)) == ("libgfortran.3.4a", nothing) - @test_throws ArgumentError BinaryProvider.parse_dl_name_version("libgfortran", MacOS(:x86_64)) - @test BinaryProvider.parse_dl_name_version("libgfortran.so", Linux(:x86_64)) == ("libgfortran", nothing) - @test BinaryProvider.parse_dl_name_version("libgfortran.so.3", Linux(:x86_64)) == ("libgfortran", v"3") - @test BinaryProvider.parse_dl_name_version("libgfortran.so.3.4", Linux(:x86_64)) == ("libgfortran", v"3.4") - @test_throws ArgumentError BinaryProvider.parse_dl_name_version("libgfortran.so.3.4a", Linux(:x86_64)) - @test_throws ArgumentError BinaryProvider.parse_dl_name_version("libgfortran", Linux(:x86_64)) - - for p in [Windows(:i686), Linux(:armv7l, :musl), FreeBSD(:x86_64), MacOS(compiler_abi=CompilerABI(:gcc4))] - fakepath = "/path/to/nowhere/Thingo.v1.2.3." * triplet(p) * ".tar.gz" - @test extract_platform_key(fakepath) == p - @test extract_name_version_platform_key(fakepath) == ("Thingo", v"1.2.3", p) - end - - # This failed on me a while back. NEVER AGAIN. - @test extract_name_version_platform_key("libjpeg.v9.0.0-b.x86_64-apple-darwin14.tar.gz") == ("libjpeg", v"9.0.0-b", MacOS()) - end - - @testset "Sys.is* overloading" begin - # Test that we can indeed ask if something is linux or windows, etc... - @test Sys.islinux(Linux(:aarch64)) - @test !Sys.islinux(Windows(:x86_64)) - @test Sys.iswindows(Windows(:i686)) - @test !Sys.iswindows(Linux(:x86_64)) - @test Sys.isapple(MacOS()) - @test !Sys.isapple(Linux(:powerpc64le)) - @test Sys.isbsd(MacOS()) - @test Sys.isbsd(FreeBSD(:x86_64)) - @test !Sys.isbsd(Linux(:powerpc64le, :musl)) - end - - @testset "Compiler ABI detection" begin - @test BinaryProvider.detect_libgfortran_abi("libgfortran.so.5", Linux(:x86_64)) == :gcc8 - @test BinaryProvider.detect_libgfortran_abi("libgfortran.4.dylib", MacOS()) == :gcc7 - @test BinaryProvider.detect_libgfortran_abi("libgfortran-3.dll", Windows(:x86_64)) == :gcc4 - @test BinaryProvider.detect_libgfortran_abi("blah.so", Linux(:aarch64)) == :gcc_any - end -end - @testset "Prefix" begin mktempdir() do temp_dir prefix = Prefix(temp_dir) @@ -370,8 +189,8 @@ end # Test we can run the script we dropped within this prefix. # Once again, something about Windows | busybox | Julia won't # pick this up even though the path clearly points to the file. - @test success(sh(`$(ppt_path)`)) - @test success(sh(`prefix_path_test.sh`)) + @test success(`$sh $(ppt_path)`) + @test success(`$sh -c prefix_path_test.sh`) end end @@ -381,34 +200,6 @@ end end end -@testset "Tarball listing parsing" begin - fake_7z_output = """ - 7-Zip [64] 9.20 Copyright (c) 1999-2010 Igor Pavlov 2010-11-18 - p7zip Version 9.20 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,4 CPUs) - - Listing archive: - - -- - Path = - Type = tar - - Date Time Attr Size Compressed Name - ------------------- ----- ------------ ------------ ------------------------ - 2017-04-10 14:45:00 D.... 0 0 bin - 2017-04-10 14:44:59 ..... 211 512 bin/socrates - ------------------- ----- ------------ ------------ ------------------------ - 211 512 1 files, 1 folders - """ - @test parse_7z_list(fake_7z_output) == ["bin/socrates"] - - fake_tar_output = """ - bin/ - bin/socrates - """ - @test parse_tar_list(fake_tar_output) == normpath.(["bin/socrates"]) - -end - @testset "Products" begin temp_prefix() do prefix # Test that basic satisfication is not guaranteed @@ -658,9 +449,11 @@ end @test_throws ArgumentError install(other_path, tarball_hash; prefix=prefix, ignore_platform=false) Base.rm(other_path; force=true) - # Ensure that hash mismatches throw errors + # Ensure that hash mismatches log errors fake_hash = reverse(tarball_hash) - @test_throws ErrorException install(tarball_path, fake_hash; prefix=prefix) + @test_logs (:error, r"Hash Mismatch") match_mode=:any begin + install(tarball_path, fake_hash; prefix=prefix) + end end # Get a valid platform tarball name that is not our native platform @@ -801,54 +594,19 @@ end end end -@testset "choose_download" begin - platforms = Dict( - # Typical binning test - Linux(:x86_64, compiler_abi=CompilerABI(:gcc4)) => "linux4", - Linux(:x86_64, compiler_abi=CompilerABI(:gcc7)) => "linux7", - Linux(:x86_64, compiler_abi=CompilerABI(:gcc8)) => "linux8", - - # Ambiguity test - Linux(:aarch64, compiler_abi=CompilerABI(:gcc4)) => "linux4", - Linux(:aarch64, compiler_abi=CompilerABI(:gcc5)) => "linux5", - - MacOS(:x86_64, compiler_abi=CompilerABI(:gcc4)) => "mac4", - Windows(:x86_64, compiler_abi=CompilerABI(:gcc_any, :cxx11)) => "win", - ) - - @test choose_download(platforms, Linux(:x86_64)) == "linux8" - @test choose_download(platforms, Linux(:x86_64, compiler_abi=CompilerABI(:gcc7))) == "linux7" - - # Ambiguity test - @test choose_download(platforms, Linux(:aarch64)) == "linux5" - @test choose_download(platforms, Linux(:aarch64; compiler_abi=CompilerABI(:gcc4))) == "linux5" - @test choose_download(platforms, Linux(:aarch64; compiler_abi=CompilerABI(:gcc5))) == "linux5" - @test choose_download(platforms, Linux(:aarch64; compiler_abi=CompilerABI(:gcc6))) == "linux5" - @test choose_download(platforms, Linux(:aarch64; compiler_abi=CompilerABI(:gcc7))) == nothing - - @test choose_download(platforms, MacOS(:x86_64)) == "mac4" - @test choose_download(platforms, MacOS(:x86_64, compiler_abi=CompilerABI(:gcc7))) == nothing - - @test choose_download(platforms, Windows(:x86_64, compiler_abi=CompilerABI(:gcc_any, :cxx11))) == "win" - @test choose_download(platforms, Windows(:x86_64, compiler_abi=CompilerABI(:gcc_any, :cxx03))) == nothing - - # Poor little guy - @test choose_download(platforms, FreeBSD(:x86_64)) == nothing -end - # Use `build_libfoo_tarball.jl` in the BinaryBuilder.jl repository to generate more of these const bin_prefix = "https://github.com/staticfloat/small_bin/raw/51b13b44feb2a262e2e04690bfa54d03167533f2/libfoo" const libfoo_downloads = Dict( - Linux(:aarch64, :glibc) => ("$bin_prefix/libfoo.aarch64-linux-gnu.tar.gz", "36886ac25cf5678c01fe20630b413f9354b7a3721c6a2c2043162f7ebd147ff5"), - Linux(:armv7l, :glibc) => ("$bin_prefix/libfoo.arm-linux-gnueabihf.tar.gz", "147ebaeb1a722da43ee08705689aed71ac87c3c2c907af047c6721c0025ba383"), - Linux(:powerpc64le, :glibc) => ("$bin_prefix/libfoo.powerpc64le-linux-gnu.tar.gz", "5c35295ac161272ada9a77d1f6b770e30ea864e521e31853258cbc36ad4c4468"), - Linux(:i686, :glibc) => ("$bin_prefix/libfoo.i686-linux-gnu.tar.gz", "97655b6a218d61284723b6923d7c96e6a256fa68b9419d723c588aa24404b102"), - Linux(:x86_64, :glibc) => ("$bin_prefix/libfoo.x86_64-linux-gnu.tar.gz", "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391"), + Linux(:aarch64; libc=:glibc) => ("$bin_prefix/libfoo.aarch64-linux-gnu.tar.gz", "36886ac25cf5678c01fe20630b413f9354b7a3721c6a2c2043162f7ebd147ff5"), + Linux(:armv7l; libc=:glibc) => ("$bin_prefix/libfoo.arm-linux-gnueabihf.tar.gz", "147ebaeb1a722da43ee08705689aed71ac87c3c2c907af047c6721c0025ba383"), + Linux(:powerpc64le; libc=:glibc) => ("$bin_prefix/libfoo.powerpc64le-linux-gnu.tar.gz", "5c35295ac161272ada9a77d1f6b770e30ea864e521e31853258cbc36ad4c4468"), + Linux(:i686; libc=:glibc) => ("$bin_prefix/libfoo.i686-linux-gnu.tar.gz", "97655b6a218d61284723b6923d7c96e6a256fa68b9419d723c588aa24404b102"), + Linux(:x86_64; libc=:glibc) => ("$bin_prefix/libfoo.x86_64-linux-gnu.tar.gz", "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391"), - Linux(:aarch64, :musl) => ("$bin_prefix/libfoo.aarch64-linux-musl.tar.gz", "81751477c1e3ee6c93e1c28ee7db2b99d1eed0d6ce86dc30d64c2e5dd4dfe88d"), - Linux(:armv7l, :musl) => ("$bin_prefix/libfoo.arm-linux-musleabihf.tar.gz", "bb65aad58f2e6fc39dc9688da1bca5e8103a3a3fa67dc589debbd2e98176f0e1"), - Linux(:i686, :musl) => ("$bin_prefix/libfoo.i686-linux-musl.tar.gz", "5f02fd1fe19f3a565fb128d3673b35c7b3214a101cef9dcbb202c0092438a87b"), - Linux(:x86_64, :musl) => ("$bin_prefix/libfoo.x86_64-linux-musl.tar.gz", "ea630600a12d2c1846bc93bcc8d9638a4991f63329205c534d93e0a3de5f641d"), + Linux(:aarch64; libc=:musl) => ("$bin_prefix/libfoo.aarch64-linux-musl.tar.gz", "81751477c1e3ee6c93e1c28ee7db2b99d1eed0d6ce86dc30d64c2e5dd4dfe88d"), + Linux(:armv7l; libc=:musl) => ("$bin_prefix/libfoo.arm-linux-musleabihf.tar.gz", "bb65aad58f2e6fc39dc9688da1bca5e8103a3a3fa67dc589debbd2e98176f0e1"), + Linux(:i686; libc=:musl) => ("$bin_prefix/libfoo.i686-linux-musl.tar.gz", "5f02fd1fe19f3a565fb128d3673b35c7b3214a101cef9dcbb202c0092438a87b"), + Linux(:x86_64; libc=:musl) => ("$bin_prefix/libfoo.x86_64-linux-musl.tar.gz", "ea630600a12d2c1846bc93bcc8d9638a4991f63329205c534d93e0a3de5f641d"), FreeBSD(:x86_64) => ("$bin_prefix/libfoo.x86_64-unknown-freebsd11.1.tar.gz", "5f6edd6247b3685fa5c42c98a53d2a3e1eef6242c2bb3cdbb5fe23f538703fe4"), MacOS(:x86_64) => ("$bin_prefix/libfoo.x86_64-apple-darwin14.tar.gz", "fcc268772d6f21d65b45fcf3854a3142679b78e53c7673dac26c95d6ccc89a24"), @@ -867,7 +625,7 @@ const libfoo_downloads = Dict( @test !BinaryProvider.safe_isfile("http://google.com") url, hash = try - choose_download(libfoo_downloads, platform) + select_platform(libfoo_downloads, platform) catch e if isa(e, ArgumentError) @warn("Platform $platform does not have a libfoo download, skipping download tests")