From 623625b843b0e01070e936d45787873370176153 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 29 Mar 2021 17:30:40 +0200 Subject: [PATCH] allow option for using git CLI for git downloads (#2448) allow an option for using git CLI for git downloads --- CHANGELOG.md | 2 ++ src/GitTools.jl | 21 +++++++++++++++++---- src/Operations.jl | 2 +- src/Types.jl | 2 +- test/new.jl | 40 ++++++++++++++++++++++------------------ test/pkg.jl | 4 ++-- 6 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a220f46fb..11d9d8965f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ Pkg v1.7 Release Notes ====================== +- It is now possible to use an external `git` executable instead of the default libgit2 library for + the downloads that happen via the Git protocol by setting the environment variable `JULIA_PKG_USE_CLI_GIT=true`. - Registries downloaded from Pkg Server (not git) is now assumed to be immutable. Manual changes to their files might not be picked up by a running Pkg session. - Adding packages by folder name in the REPL mode now requires a prepending a `./` to the folder name package folder is in the current folder, e.g. `add ./Package` is required instead of `add Pacakge`. This is to avoid confusion between the package name `Package` and the local directory `Package`. - `rm`, `pin`, and `free` now support the `--all` option, and the api variants gain the `all_pkgs::Bool` kwarg, to perform the operation on all packages within the project or manifest, depending on the mode of the operation. diff --git a/src/GitTools.jl b/src/GitTools.jl index 978da47f24..9ff08eceff 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -10,6 +10,8 @@ import Base: SHA1 import LibGit2 using Printf +use_cli_git() = get(ENV, "JULIA_PKG_USE_CLI_GIT", "") == "true" + function transfer_progress(progress::Ptr{LibGit2.TransferProgress}, p::Any) progress = unsafe_load(progress) @assert haskey(p, :transfer_progress) @@ -104,9 +106,14 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw if credentials === nothing credentials = LibGit2.CachedCredentials() end - mkpath(source_path) try - return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...) + if use_cli_git() + run(`git clone --quiet $url $source_path`) + return LibGit2.GitRepo(source_path) + else + mkpath(source_path) + return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...) + end catch err rm(source_path; force=true, recursive=true) err isa LibGit2.GitError || err isa InterruptException || rethrow() @@ -124,7 +131,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw end end -function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, kwargs...) +function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, refspecs=[""], kwargs...) if remoteurl === nothing remoteurl = LibGit2.with(LibGit2.get(LibGit2.GitRemote, repo, "origin")) do remote LibGit2.url(remote) @@ -150,7 +157,13 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials = LibGit2.CachedCredentials() end try - return LibGit2.fetch(repo; remoteurl=remoteurl, callbacks=callbacks, kwargs...) + if use_cli_git() + cd(LibGit2.path(repo)) do + run(`git fetch -q $remoteurl $(only(refspecs))`) + end + else + return LibGit2.fetch(repo; remoteurl=remoteurl, callbacks=callbacks, refspecs=refspecs, kwargs...) + end catch err err isa LibGit2.GitError || rethrow() if (err.class == LibGit2.Error.Repository && err.code == LibGit2.Error.ERROR) diff --git a/src/Operations.jl b/src/Operations.jl index 007da6c98e..338d6e48ce 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -666,7 +666,7 @@ function download_source(ctx::Context; readonly=true) for i in 1:ctx.num_concurrent_downloads @async begin for (pkg, urls, path) in jobs - if ctx.use_libgit2_for_all_downloads + if ctx.use_git_for_all_downloads put!(results, (pkg, false, (urls, path))) continue end diff --git a/src/Types.jl b/src/Types.jl index 81d5f87df7..3fb7ad919c 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -320,7 +320,7 @@ include("manifest.jl") Base.@kwdef mutable struct Context env::EnvCache = EnvCache() io::IO = something(DEFAULT_IO[]) - use_libgit2_for_all_downloads::Bool = false + use_git_for_all_downloads::Bool = false use_only_tarballs_for_downloads::Bool = false num_concurrent_downloads::Int = 8 diff --git a/test/new.jl b/test/new.jl index 4eb665727a..3e0463dc1e 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2321,24 +2321,28 @@ end # # Note: these tests should be run on clean depots @testset "downloads" begin - # libgit2 downloads - isolate() do - Pkg.add("Example"; use_libgit2_for_all_downloads=true) - @test haskey(Pkg.dependencies(), exuuid) - @eval import $(Symbol(TEST_PKG.name)) - @test_throws SystemError open(pathof(eval(Symbol(TEST_PKG.name))), "w") do io end # check read-only - Pkg.rm(TEST_PKG.name) - end - isolate() do - @testset "libgit2 downloads" begin - Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true) - @test haskey(Pkg.dependencies(), TEST_PKG.uuid) - Pkg.rm(TEST_PKG.name) - end - @testset "tarball downloads" begin - Pkg.add("JSON"; use_only_tarballs_for_downloads=true) - @test "JSON" in [pkg.name for (uuid, pkg) in Pkg.dependencies()] - Pkg.rm("JSON") + for v in (nothing, "true") + withenv("JULIA_PKG_USE_CLI_GIT" => v) do + # libgit2 downloads + isolate() do + Pkg.add("Example"; use_git_for_all_downloads=true) + @test haskey(Pkg.dependencies(), exuuid) + @eval import $(Symbol(TEST_PKG.name)) + @test_throws SystemError open(pathof(eval(Symbol(TEST_PKG.name))), "w") do io end # check read-only + Pkg.rm(TEST_PKG.name) + end + isolate() do + @testset "libgit2 downloads" begin + Pkg.add(TEST_PKG.name; use_git_for_all_downloads=true) + @test haskey(Pkg.dependencies(), TEST_PKG.uuid) + Pkg.rm(TEST_PKG.name) + end + @testset "tarball downloads" begin + Pkg.add("JSON"; use_only_tarballs_for_downloads=true) + @test "JSON" in [pkg.name for (uuid, pkg) in Pkg.dependencies()] + Pkg.rm("JSON") + end + end end end end diff --git a/test/pkg.jl b/test/pkg.jl index 92930c0887..c19719d0c3 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -378,7 +378,7 @@ end temp_pkg_dir() do project_path @testset "libgit2 downloads" begin - Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true) + Pkg.add(TEST_PKG.name; use_git_for_all_downloads=true) @test haskey(Pkg.dependencies(), TEST_PKG.uuid) @eval import $(Symbol(TEST_PKG.name)) @test_throws SystemError open(pathof(eval(Symbol(TEST_PKG.name))), "w") do io end # check read-only @@ -400,7 +400,7 @@ end temp_pkg_dir() do project_path @testset "libgit2 downloads" begin - Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true) + Pkg.add(TEST_PKG.name; use_git_for_all_downloads=true) @test haskey(Pkg.dependencies(), TEST_PKG.uuid) Pkg.rm(TEST_PKG.name) end