Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 108 additions & 79 deletions pkgs/build-support/fetchgit/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
git,
git-lfs,
cacert,
callPackage,
}:

let
Expand Down Expand Up @@ -64,6 +65,9 @@ lib.makeOverridable (
fetchTags ? false,
# make this subdirectory the root of the result
rootDir ? "",
publicKeys ? [ ],
verifyCommit ? false,
verifyTag ? false,
}:

/*
Expand Down Expand Up @@ -108,89 +112,114 @@ lib.makeOverridable (
else
# FIXME fetching HEAD if no rev or tag is provided is problematic at best
"HEAD";
fetchresult =
if builtins.isString sparseCheckout then
# Changed to throw on 2023-06-04
throw
"Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more."
else
stdenvNoCC.mkDerivation {
inherit name;

builder = ./builder.sh;
fetcher = ./nix-prefetch-git;

nativeBuildInputs = [
git
cacert
]
++ lib.optionals fetchLFS [ git-lfs ]
++ nativeBuildInputs;

inherit outputHash outputHashAlgo;
outputHashMode = "recursive";

# git-sparse-checkout(1) says:
# > When the --stdin option is provided, the directories or patterns are read
# > from standard in as a newline-delimited list instead of from the arguments.
sparseCheckout = builtins.concatStringsSep "\n" sparseCheckout;

leaveDotGit = leaveDotGit || verifyCommit || verifyTag;

inherit
url
fetchLFS
fetchSubmodules
deepClone
branchName
nonConeMode
preFetch
postFetch
fetchTags
rootDir
;
rev = revWithTag;

NIX_PREFETCH_GIT_CHECKOUT_HOOK =
if verifyTag then
''
clean_git -C "$dir" fetch origin "$rev:$rev"
''
else
null;

postHook =
if netrcPhase == null then
null
else
''
${netrcPhase}
# required that git uses the netrc file
mv {,.}netrc
export NETRC=$PWD/.netrc
export HOME=$PWD
'';

impureEnvVars =
lib.fetchers.proxyImpureEnvVars
++ netrcImpureEnvVars
++ [
"GIT_PROXY_COMMAND"
"NIX_GIT_SSL_CAINFO"
"SOCKS_SERVER"

# This is a parameter intended to be set by setup hooks or preFetch
# scripts that want per-URL control over HTTP proxies used by Git
# (if per-URL control isn't needed, `http_proxy` etc. will
# suffice). It must be a whitespace-separated (with backslash as an
# escape character) list of pairs like this:
#
# http://domain1/path1 proxy1 https://domain2/path2 proxy2
#
# where the URLs are as documented in the `git-config` manual page
# under `http.<url>.*`, and the proxies are as documented on the
# same page under `http.proxy`.
"FETCHGIT_HTTP_PROXIES"
];

inherit preferLocalBuild meta allowedRequisites;

passthru = {
gitRepoUrl = url;
inherit tag;
};
};

verifySignature = callPackage ./verify.nix { };
in

if builtins.isString sparseCheckout then
# Changed to throw on 2023-06-04
throw
"Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more."
else
stdenvNoCC.mkDerivation {
inherit name;

builder = ./builder.sh;
fetcher = ./nix-prefetch-git;

nativeBuildInputs = [
git
cacert
]
++ lib.optionals fetchLFS [ git-lfs ]
++ nativeBuildInputs;

inherit outputHash outputHashAlgo;
outputHashMode = "recursive";

# git-sparse-checkout(1) says:
# > When the --stdin option is provided, the directories or patterns are read
# > from standard in as a newline-delimited list instead of from the arguments.
sparseCheckout = builtins.concatStringsSep "\n" sparseCheckout;

if verifyCommit || verifyTag then
verifySignature {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there should be documentation, probably here but definitely somewhere in these changes, to explain the need for the two-step verification. Something like the explanation at #330457 (comment) (notwithstanding my overall review comment) needs to be included in the repository itself, rather than needing someone to go looking for it in the PR discussions.

inherit
url
revWithTag
verifyCommit
verifyTag
publicKeys
leaveDotGit
fetchLFS
fetchSubmodules
deepClone
branchName
nonConeMode
preFetch
postFetch
fetchTags
rootDir
fetchresult
name
;
rev = revWithTag;

postHook =
if netrcPhase == null then
null
else
''
${netrcPhase}
# required that git uses the netrc file
mv {,.}netrc
export NETRC=$PWD/.netrc
export HOME=$PWD
'';

impureEnvVars =
lib.fetchers.proxyImpureEnvVars
++ netrcImpureEnvVars
++ [
"GIT_PROXY_COMMAND"
"NIX_GIT_SSL_CAINFO"
"SOCKS_SERVER"

# This is a parameter intended to be set by setup hooks or preFetch
# scripts that want per-URL control over HTTP proxies used by Git
# (if per-URL control isn't needed, `http_proxy` etc. will
# suffice). It must be a whitespace-separated (with backslash as an
# escape character) list of pairs like this:
#
# http://domain1/path1 proxy1 https://domain2/path2 proxy2
#
# where the URLs are as documented in the `git-config` manual page
# under `http.<url>.*`, and the proxies are as documented on the
# same page under `http.proxy`.
"FETCHGIT_HTTP_PROXIES"
];

inherit preferLocalBuild meta allowedRequisites;

passthru = {
gitRepoUrl = url;
inherit tag;
};
}
else
fetchresult
)
)
55 changes: 54 additions & 1 deletion pkgs/build-support/fetchgit/tests.nix
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
{ testers, fetchgit, ... }:
{
testers,
fetchgit,
fetchurl,
...
}:
{
simple = testers.invalidateFetcherByDrvHash fetchgit {
name = "simple-nix-source";
Expand Down Expand Up @@ -105,4 +110,52 @@
rootDir = "misc/systemd";
sha256 = "sha256-UhxHk4SrXYq7ZDMtXLig5SigpbITrVgkpFTmryuvpcM=";
};

ssh-commit-verification = testers.invalidateFetcherByDrvHash fetchgit {
name = "ssh-commit-verification-source";
url = "https://codeberg.org/flandweber/git-verify";
rev = "a43858e8f106b313aed68b6455a45340db7dd758";
sha256 = "sha256-IH2ed4oRruhWkPorcEETmbzWpaqY6/tNSUUMk+ntZ3M=";
verifyCommit = true;
publicKeys = [
{
type = "ssh-ed25519";
key = "AAAAC3NzaC1lZDI1NTE5AAAAIBiNDWMPZNRItkm1U1CQkJUUrrmM+l7CdE6wyUHzr4Nr";
}
];
};

gpg-commit-verification = testers.invalidateFetcherByDrvHash fetchgit {
name = "gpg-commit-verification-source";
url = "https://gitlab.torproject.org/tpo/core/tor";
rev = "8888e4ca6b44bb7476139be4644e739035441b35";
sha256 = "sha256-HSYUwzm3k4GAIt/ds80i8HM8hZLgC7zTUJBqhF6wBvA=";
verifyCommit = true;
publicKeys = [
{
type = "gpg";
key = fetchurl {
url = "https://web.archive.org/web/20241109193517/https://keys.openpgp.org/vks/v1/by-fingerprint/5EF3A41171BB77E6110ED2D01F3D03348DB1A3E2";
sha256 = "sha256-xvBWfaS1py7vyDIIYGtATqBOnWafd3B6OB2Blhfm4MU=";
};
}
];
};

gpg-tag-verification = testers.invalidateFetcherByDrvHash fetchgit {
name = "gpg-tag-verification-source";
url = "https://gitlab.torproject.org/tpo/core/tor";
tag = "tor-0.4.8.12";
sha256 = "sha256-AXVD5I7KyDVAPIOHcPRHHfW0uwPxjCuY9t1Bf/pBLps=";
verifyTag = true;
publicKeys = [
{
type = "gpg";
key = fetchurl {
url = "https://web.archive.org/web/20241109193821/https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36";
sha256 = "sha256-M4mvelY1nLeGuhgZIpF4oAe80kbJl2+wcDI6zp9YwXo=";
};
}
];
};
}
89 changes: 89 additions & 0 deletions pkgs/build-support/fetchgit/verify.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
{
lib,
runCommand,
writeShellApplication,
writeText,
git,
openssh,
gnupg,
}:
(
{
name,
revWithTag,
verifyCommit,
verifyTag,
publicKeys,
leaveDotGit,
fetchresult,
}:
let
# split gpg keys from ssh keys
keysPartitioned = lib.partition (k: k.type == "gpg") publicKeys;
gpgKeys = lib.catAttrs "key" keysPartitioned.right;
sshKeys = keysPartitioned.wrong;
Comment on lines +23 to +24
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think these are confusingly inconsistent:

  • gpgKeys is a list of store paths, where the paths are the result of the derivations in the keys attribute of the given publicKeys attrsets.
  • sshKeys is a list of attrsets, with keys stored as strings in the key attribute of each attrset.

To fix this, I think (a) there should be different names for values that are expected to be paths versus strings, e.g. use key in SSH keys and keypath in GPG keys, and (b) the values in gpgKeys shouldn't be broken out until they're used, as with sshKeys, or sshKeys should be renamed to something that's clearly different like sshKeyAttrs.

# create a keyring containing gpgKeys
gpgKeyring = runCommand "gpgKeyring" { buildInputs = [ gnupg ]; } ''
gpg --homedir /build --no-default-keyring --keyring $out --fingerprint # create empty keyring at $out
for KEY in ${lib.concatStringsSep " " gpgKeys}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gpgKeys is going to be a list of store paths, so I think this should be lib.escapeShellArgs gpgKeys.

do
gpg --homedir /build --no-default-keyring --keyring $out --import $KEY # import $KEY
done
'';
gitOnRepo = writeShellApplication {
name = "gitOnRepo";
runtimeInputs = [ git ];
text = ''
git -C "${fetchresult}" -c safe.directory='*' "$@"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: you're quoting a store path here, but not in other bits of shell code. I don't particularly mind which you do (personally, I'd either pass things in as environment variables then quote the variable, or use lib.escapeShellArg again), but consistency across this file would be nice.

'';
};
# wrap gpg to use gpgKeyring
gpgWithKeys = writeShellApplication {
name = "gpgWithKeys";
runtimeInputs = [
gnupg
gitOnRepo
];
text = ''
committerTime="$(gitOnRepo -c core.pager=cat log --format="%cd" --date=raw -n 1 ${revWithTag})"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

core.pager shouldn't need to be set here; I think that adds complexity for no gain.

gpg --faked-system-time "$committerTime" --homedir /build --no-default-keyring --always-trust --keyring ${gpgKeyring} "$@"
'';
};
# create "allowed signers" file for ssh key verification: https://man.openbsd.org/ssh-keygen.1#ALLOWED_SIGNERS
allowedSignersFile = writeText "allowed signers" (
lib.concatMapStrings (k: "* ${k.type} ${k.key}\n") sshKeys
);
in
runCommand name
{
buildInputs = [
gitOnRepo
openssh
gpgWithKeys
];
inherit verifyCommit verifyTag leaveDotGit;
}
''
gpgWithKeys -k
if test "$verifyCommit" == 1; then
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test "$verifyCommit" == 1 feels very fragile to me: it only works because runCommand converts true to the string 1 and false to the null string.

I'd much prefer [[ "$verifyCommit" ]]. Which does depend on false becoming the null string, but that's (IME) a much more common shell idiom, so I think is much more likely to hold indefinitely.

gitOnRepo \
-c gpg.ssh.allowedSignersFile="${allowedSignersFile}" \
-c gpg.program="gpgWithKeys" \
verify-commit ${revWithTag}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think ${revWithTag} needs to be ${lib.escapeShellArg revWithTag}: it's rare, but I'm fairly sure it's entirely valid for tags and branch names and so forth to contain spaces or other characters that need to be escaped in shell.

fi

if test "$verifyTag" == 1; then
gitOnRepo \
-c gpg.ssh.allowedSignersFile="${allowedSignersFile}" \
-c gpg.program="gpgWithKeys" \
verify-tag ${revWithTag}
fi

if test "$leaveDotGit" != 1; then
cp -r --no-preserve=all "${fetchresult}" $out
rm -rf "$out"/.git
else
ln "${fetchresult}" $out
fi
''
)
Loading