allow option for using git CLI for git downloads (#2448)

allow an option for using git CLI for git downloads

CHANGELOG.md

@@ -1,6 +1,8 @@
 Pkg v1.7 Release Notes
 ======================
 
+- It is now possible to use an external `git` executable instead of the default libgit2 library for 
+  the downloads that happen via the Git protocol by setting the environment variable `JULIA_PKG_USE_CLI_GIT=true`.
 - Registries downloaded from Pkg Server (not git) is now assumed to be immutable. Manual changes to their files might not be picked up by a running Pkg session.
 - Adding packages by folder name in the REPL mode now requires a prepending a `./` to the folder name package folder is in the current folder, e.g. `add ./Package` is required instead of `add Pacakge`. This is to avoid confusion between the package name `Package` and the local directory `Package`.
 - `rm`, `pin`, and `free` now support the `--all` option, and the api variants gain the `all_pkgs::Bool` kwarg, to perform the operation on all packages within the project or manifest, depending on the mode of the operation.

src/GitTools.jl

@@ -10,6 +10,8 @@ import Base: SHA1
 import LibGit2
 using Printf
 
+use_cli_git() = get(ENV, "JULIA_PKG_USE_CLI_GIT", "") == "true"
+
 function transfer_progress(progress::Ptr{LibGit2.TransferProgress}, p::Any)
     progress = unsafe_load(progress)
     @assert haskey(p, :transfer_progress)
@@ -104,9 +106,14 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw
     if credentials === nothing
         credentials = LibGit2.CachedCredentials()
     end
-    mkpath(source_path)
     try
-        return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...)
+        if use_cli_git()
+            run(`git clone --quiet $url $source_path`)
+            return LibGit2.GitRepo(source_path)
+        else
+            mkpath(source_path)
+            return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...)
+        end
     catch err
         rm(source_path; force=true, recursive=true)
         err isa LibGit2.GitError || err isa InterruptException || rethrow()
@@ -124,7 +131,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw
     end
 end
 
-function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, kwargs...)
+function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, refspecs=[""], kwargs...)
     if remoteurl === nothing
         remoteurl = LibGit2.with(LibGit2.get(LibGit2.GitRemote, repo, "origin")) do remote
             LibGit2.url(remote)
@@ -150,7 +157,13 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing,
         credentials = LibGit2.CachedCredentials()
     end
     try
-        return LibGit2.fetch(repo; remoteurl=remoteurl, callbacks=callbacks, kwargs...)
+        if use_cli_git()
+            cd(LibGit2.path(repo)) do
+                run(`git fetch -q $remoteurl $(only(refspecs))`)
+            end
+        else
+            return LibGit2.fetch(repo; remoteurl=remoteurl, callbacks=callbacks, refspecs=refspecs, kwargs...)
+        end
     catch err
         err isa LibGit2.GitError || rethrow()
         if (err.class == LibGit2.Error.Repository && err.code == LibGit2.Error.ERROR)

src/Operations.jl

@@ -666,7 +666,7 @@ function download_source(ctx::Context; readonly=true)
         for i in 1:ctx.num_concurrent_downloads
             @async begin
                 for (pkg, urls, path) in jobs
-                    if ctx.use_libgit2_for_all_downloads
+                    if ctx.use_git_for_all_downloads
                         put!(results, (pkg, false, (urls, path)))
                         continue
                     end

src/Types.jl

@@ -320,7 +320,7 @@ include("manifest.jl")
 Base.@kwdef mutable struct Context
     env::EnvCache = EnvCache()
     io::IO = something(DEFAULT_IO[])
-    use_libgit2_for_all_downloads::Bool = false
+    use_git_for_all_downloads::Bool = false
     use_only_tarballs_for_downloads::Bool = false
     num_concurrent_downloads::Int = 8
 

test/new.jl

@@ -2321,24 +2321,28 @@ end
 #
 # Note: these tests should be run on clean depots
 @testset "downloads" begin
-    # libgit2 downloads
-    isolate() do
-        Pkg.add("Example"; use_libgit2_for_all_downloads=true)
-        @test haskey(Pkg.dependencies(), exuuid)
-        @eval import $(Symbol(TEST_PKG.name))
-        @test_throws SystemError open(pathof(eval(Symbol(TEST_PKG.name))), "w") do io end  # check read-only
-        Pkg.rm(TEST_PKG.name)
-    end
-    isolate() do
-        @testset "libgit2 downloads" begin
-            Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true)
-            @test haskey(Pkg.dependencies(), TEST_PKG.uuid)
-            Pkg.rm(TEST_PKG.name)
-        end
-        @testset "tarball downloads" begin
-            Pkg.add("JSON"; use_only_tarballs_for_downloads=true)
-            @test "JSON" in [pkg.name for (uuid, pkg) in Pkg.dependencies()]
-            Pkg.rm("JSON")
+    for v in (nothing, "true")
+        withenv("JULIA_PKG_USE_CLI_GIT" => v) do
+            # libgit2 downloads
+            isolate() do
+                Pkg.add("Example"; use_git_for_all_downloads=true)
+                @test haskey(Pkg.dependencies(), exuuid)
+                @eval import $(Symbol(TEST_PKG.name))
+                @test_throws SystemError open(pathof(eval(Symbol(TEST_PKG.name))), "w") do io end  # check read-only
+                Pkg.rm(TEST_PKG.name)
+            end
+            isolate() do
+                @testset "libgit2 downloads" begin
+                    Pkg.add(TEST_PKG.name; use_git_for_all_downloads=true)
+                    @test haskey(Pkg.dependencies(), TEST_PKG.uuid)
+                    Pkg.rm(TEST_PKG.name)
+                end
+                @testset "tarball downloads" begin
+                    Pkg.add("JSON"; use_only_tarballs_for_downloads=true)
+                    @test "JSON" in [pkg.name for (uuid, pkg) in Pkg.dependencies()]
+                    Pkg.rm("JSON")
+                end
+            end
         end
     end
 end

test/pkg.jl

@@ -378,7 +378,7 @@ end
 
 temp_pkg_dir() do project_path
     @testset "libgit2 downloads" begin
-        Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true)
+        Pkg.add(TEST_PKG.name; use_git_for_all_downloads=true)
         @test haskey(Pkg.dependencies(), TEST_PKG.uuid)
         @eval import $(Symbol(TEST_PKG.name))
         @test_throws SystemError open(pathof(eval(Symbol(TEST_PKG.name))), "w") do io end  # check read-only
@@ -400,7 +400,7 @@ end
 
 temp_pkg_dir() do project_path
     @testset "libgit2 downloads" begin
-        Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true)
+        Pkg.add(TEST_PKG.name; use_git_for_all_downloads=true)
         @test haskey(Pkg.dependencies(), TEST_PKG.uuid)
         Pkg.rm(TEST_PKG.name)
     end