Skip to content

Commit

Permalink
fetchTree/fetchGit: re-enable shallow fetching
Browse files Browse the repository at this point in the history
Add several tests for git fetching:
- shallow-cache-separation: can fetch the same repo shallowly and non-shallowly
- shallow-ignore-ref: ensure that ref gets ignored when shallow=true is set
- ssh-shallow: can fetch a git repo via ssh using shallow=1
  • Loading branch information
DavHau committed Jan 19, 2024
1 parent d762caf commit bc00fa4
Show file tree
Hide file tree
Showing 6 changed files with 168 additions and 8 deletions.
9 changes: 8 additions & 1 deletion src/libfetchers/git-utils.cc
Original file line number Diff line number Diff line change
Expand Up @@ -387,13 +387,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
// then use code that was removed in this commit (see blame)

auto dir = this->path;
Strings gitArgs;
if (shallow) {
gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
}
else {
gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec };
}

runProgram(RunOptions {
.program = "git",
.searchPath = true,
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
.args = { "-C", path.abs(), "fetch", "--quiet", "--force", "--", url, refspec },
.args = gitArgs,
.input = {},
.isInteractive = true
});
Expand Down
16 changes: 10 additions & 6 deletions src/libfetchers/git.cc
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ bool touchCacheFile(const Path & path, time_t touch_time)
return lutimes(path.c_str(), times) == 0;
}

Path getCachePath(std::string_view key)
Path getCachePath(std::string_view key, bool shallow)
{
return getCacheDir() + "/nix/gitv3/" +
hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false);
return getCacheDir()
+ "/nix/gitv3/"
+ hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false)
+ (shallow ? "-shallow" : "");
}

// Returns the name of the HEAD branch.
Expand Down Expand Up @@ -92,7 +94,8 @@ std::optional<std::string> readHead(const Path & path)
// Persist the HEAD ref from the remote repo in the local cached repo.
bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
{
Path cacheDir = getCachePath(actualUrl);
// set shallow=false as HEAD will never be queried for a shallow repo
Path cacheDir = getCachePath(actualUrl, false);
try {
runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) {
Expand All @@ -107,7 +110,8 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
Path cacheDir = getCachePath(actualUrl);
// set shallow=false as HEAD will never be queried for a shallow repo
Path cacheDir = getCachePath(actualUrl, false);
Path headRefFile = cacheDir + "/HEAD";

time_t now = time(0);
Expand Down Expand Up @@ -508,7 +512,7 @@ struct GitInputScheme : InputScheme
if (!input.getRev())
input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
} else {
Path cacheDir = getCachePath(repoInfo.url);
Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input));
repoDir = cacheDir;
repoInfo.gitDir = ".";

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
{
description = "can fetch the same repo shallowly and non-shallowly";
script = ''
# create branch1 off of main
client.succeed(f"""
echo chiang-mai > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit1' \
\
&& {repo.git} push origin --all
""")
# save the revision
mainRev = client.succeed(f"""
{repo.git} rev-parse main
""").strip()
# fetch shallowly
revCountShallow = client.succeed(f"""
nix eval --impure --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{mainRev}";
shallow = true;
}}).revCount
'
""").strip()
# ensure the revCount is 0
assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}"
# fetch non-shallowly
revCountNonShallow = client.succeed(f"""
nix eval --impure --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{mainRev}";
shallow = false;
}}).revCount
'
""").strip()
# ensure the revCount is 1
assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}"
# fetch shallowly again
revCountShallow2 = client.succeed(f"""
nix eval --impure --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{mainRev}";
shallow = true;
}}).revCount
'
""").strip()
# ensure the revCount is 0
assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}"
'';
}
40 changes: 40 additions & 0 deletions tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
{
description = "ensure that ref gets ignored when shallow=true is set";
script = ''
# create branch1 off of main
client.succeed(f"""
echo chiang-mai > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit1' \
\
&& {repo.git} checkout -b branch1 main \
&& echo bangkok > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit2' \
\
&& {repo.git} push origin --all
""")
# save the revisions
mainRev = client.succeed(f"""
{repo.git} rev-parse main
""").strip()
branch1Rev = client.succeed(f"""
{repo.git} rev-parse branch1
""").strip()
# Ensure that ref gets ignored when fetching shallowly.
# This would fail if the ref was respected, as branch1Rev is not on main.
client.succeed(f"""
nix eval --impure --raw --expr '
(builtins.fetchGit {{
url = "{repo.remote}";
rev = "{branch1Rev}";
ref = "main";
shallow = true;
}})
'
""")
'';
}
52 changes: 52 additions & 0 deletions tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
{
description = "can fetch a git repo via ssh using shallow=1";
script = ''
# add a file to the repo
client.succeed(f"""
echo chiang-mai > {repo.path}/thailand \
&& {repo.git} add thailand \
&& {repo.git} commit -m 'commit1'
""")
# memoize the revision
rev1 = client.succeed(f"""
{repo.git} rev-parse HEAD
""").strip()
# push to the server
client.succeed(f"""
{repo.git} push origin-ssh main
""")
fetchGit_expr = f"""
builtins.fetchGit {{
url = "{repo.remote_ssh}";
rev = "{rev1}";
shallow = true;
}}
"""
# fetch the repo via nix
fetched1 = client.succeed(f"""
nix eval --impure --raw --expr '({fetchGit_expr}).outPath'
""")
# check if the committed file is there
client.succeed(f"""
test -f {fetched1}/thailand
""")
# check if the revision is the same
rev1_fetched = client.succeed(f"""
nix eval --impure --raw --expr '({fetchGit_expr}).rev'
""").strip()
assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
# check if revCount is 1
revCount1 = client.succeed(f"""
nix eval --impure --expr '({fetchGit_expr}).revCount'
""").strip()
print(f"revCount1: {revCount1}")
assert revCount1 == '0', f"rev count is not 0 but {revCount1}"
'';
}
2 changes: 1 addition & 1 deletion tests/nixos/fetch-git/testsupport/gitea-repo.nix
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,4 @@ in
""")
'';
};
}
}

0 comments on commit bc00fa4

Please sign in to comment.