Skip to content

fetchGit: allow fetching revs which are not ancestors of HEAD without specifying a ref #3408

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
basvandijk wants to merge 1 commit into from
Closed

fetchGit: allow fetching revs which are not ancestors of HEAD without specifying a ref #3408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions doc/manual/expressions/builtins.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,28 @@ stdenv.mkDerivation { … }
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>fetchRevInsteadOfRef</term>
<listitem>
<para>
Defaults to <literal>false</literal>.
</para>

<para>
When set to <literal>true</literal> the specified
<varname>rev</varname> is fetched <emphasis>directly</emphasis>
instead of fetching the <varname>ref</varname> (which defaults to
<literal>HEAD</literal> when not specified). This is useful in
case a <varname>ref</varname> is not available.
</para>

<para>
Do note that git servers by default <link
xlink:href="https://www.git-scm.com/docs/git-config/2.25.1#Documentation/git-config.txt-uploadpackallowAnySHA1InWant">don't allow fetching revisions directly</link>
(GitHub does allow it).
</para>
</listitem>
</varlistentry>
</variablelist>

<example>
Expand Down
174 changes: 129 additions & 45 deletions src/libexpr/primops/fetchGit.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,80 @@ struct GitInfo

std::regex revRegex("^[0-9a-fA-F]{40}$");

void fetchRev(const Path cacheDir, const std::string & uri, std::string rev) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching '%s' of Git repository '%s'", rev, uri));

// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--", uri, rev });
}

void fetchRef(const Path cacheDir, const std::string & uri, std::string ref, time_t now, Path localRefFile) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching '%s' of Git repository '%s'", ref, uri));

// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", ref, ref) });

struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;

utimes(localRefFile.c_str(), times);
}

bool isAncestor(const Path cacheDir, std::string rev1, std::string rev2) {
try {
runProgram("git", true, { "-C", cacheDir, "merge-base", "--is-ancestor", rev1, rev2});
return true;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
return false;
}
else {
throw;
}
}
}

void checkAncestor(const Path cacheDir, const std::string & uri, std::string rev, std::string ref) {
time_t now = time(0);

Path localRefFile;
if (ref.compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + ref;
else
localRefFile = cacheDir + "/refs/heads/" + ref;

struct stat st;
if (stat(localRefFile.c_str(), &st) == 0 &&
isAncestor(cacheDir, rev, chomp(readFile(localRefFile)))) return;

fetchRef(cacheDir, uri, ref, now, localRefFile);
std::string refRev = chomp(readFile(localRefFile));
if (!isAncestor(cacheDir, rev, refRev)) {
throw Error("The specified rev '%s' is not an ancestor of the specified ref '%s' with revision '%s'", rev, ref, refRev);
}
}

bool revInCache(const Path cacheDir, std::string rev) {
try {
runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
return true;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
return false;
} else {
throw;
}
}
}

GitInfo exportGit(ref<Store> store, const std::string & uri,
std::optional<std::string> ref, std::string rev,
const std::string & name)
const std::string & name, bool fetchRevInsteadOfRef)
{
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
Expand Down Expand Up @@ -79,8 +150,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
ref = "HEAD"s;
}

if (!ref) ref = "HEAD"s;

if (rev != "" && !std::regex_match(rev, revRegex))
throw Error("invalid Git revision '%s'", rev);

Expand All @@ -93,54 +162,66 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
runProgram("git", true, { "init", "--bare", cacheDir });
}

Path localRefFile;
if (ref->compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + *ref;
else
localRefFile = cacheDir + "/refs/heads/" + *ref;
if (rev.empty()) {
std::string fetchedRef = ref ? *ref : "HEAD";

bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev != "") {
try {
runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
time_t now = time(0);

Path localRefFile;
if (fetchedRef.compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + fetchedRef;
else
localRefFile = cacheDir + "/refs/heads/" + fetchedRef;

struct stat st;
if (stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) {
fetchRef(cacheDir, uri, fetchedRef, now, localRefFile);
}
rev = chomp(readFile(localRefFile));
} else {
if (revInCache(cacheDir, rev)) {
if (ref) { checkAncestor(cacheDir, uri, rev, *ref); }
} else {
if (fetchRevInsteadOfRef) {
fetchRev(cacheDir, uri, rev);
if (ref) { checkAncestor(cacheDir, uri, rev, *ref); }
} else {
throw;
std::string fetchedRef = ref ? *ref : "HEAD";

time_t now = time(0);

Path localRefFile;
if (fetchedRef.compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + fetchedRef;
else
localRefFile = cacheDir + "/refs/heads/" + fetchedRef;

fetchRef(cacheDir, uri, fetchedRef, now, localRefFile);
std::string refRev = chomp(readFile(localRefFile));

if (!isAncestor(cacheDir, rev, refRev)) {
throw Error(
"The specified rev '%s' "
"is not an ancestor of ref '%s' "
"with revision '%s'. %s",
rev, fetchedRef, refRev,
ref ? "" :
"Note that 'ref' defaults to 'HEAD' when not specified. "
"So either set a correct 'ref' "
"or set 'fetchRevInsteadOfRef = true;' "
"to directly fetch the specified 'rev' "
"(which might be denied by git servers "
"which have the config option "
"'uploadpack.allowAnySHA1InWant' set to 'false')."
);
}
}
}
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
if (doFetch)
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));

// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });

struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;

utimes(localRefFile.c_str(), times);
}

// FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo;
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
gitInfo.rev = rev;
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);

printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
Expand Down Expand Up @@ -198,6 +279,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
std::optional<std::string> ref;
std::string rev;
std::string name = "source";
bool fetchRevInsteadOfRef = false;
PathSet context;

state.forceValue(*args[0]);
Expand All @@ -216,6 +298,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "fetchRevInsteadOfRef")
fetchRevInsteadOfRef = state.forceBool(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
}
Expand All @@ -230,7 +314,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
// whitelist. Ah well.
state.checkURI(url);

auto gitInfo = exportGit(state.store, url, ref, rev, name);
auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchRevInsteadOfRef);

state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
Expand Down
17 changes: 17 additions & 0 deletions tests/fetchGit.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,23 @@ echo world > $repo/hello
git -C $repo commit -m 'Bla2' -a
rev2=$(git -C $repo rev-parse HEAD)

git -C $repo checkout -b my-branch
echo universe > $repo/hello
git -C $repo commit -m 'Bla3' -a
rev3=$(git -C $repo rev-parse HEAD)

git -C $repo checkout master

# Fetch rev3 which is not an ancestor of HEAD without specifying a ref or setting `fetchRevInsteadOfRef = true` should fail.
(! nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev3\"; }).outPath")

# Fetch rev3 which is not an ancestor of HEAD without specifying a ref but setting `fetchRevInsteadOfRef = true`.
path=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev3\"; fetchRevInsteadOfRef = true; }).outPath")
[[ $(cat $path/hello) = universe ]]

# Fetching rev3 which is not an ancestor of HEAD should fail when specifying HEAD as ref.
(! nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev3\"; ref = \"HEAD\"; })")

# Fetch the default branch.
path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
[[ $(cat $path/hello) = world ]]
Expand Down