Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions doc/languages-frameworks/rust.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,104 @@ rustPlatform.buildRustPackage rec {
}
```

## Compiling non-Rust packages that include Rust code

Several non-Rust packages incorporate Rust code for performance- or
security-sensitive parts. `rustPlatform` exposes several functions and
hooks that can be used to integrate Cargo in non-Rust packages.

### Vendoring of dependencies

Since network access is not allowed in sandboxed builds, Rust crate
dependencies need to be retrieved using a fetcher. `rustPlatform`
provides the `fetchCargoTarball` fetcher, which vendors all
dependencies of a crate. This fetcher can be used jointly with
`cargoSetupHook` to vendor dependencies in derivations that do not use
`buildRustPackage`.

In the following partial example, `fetchCargoTarball` and
`cargoSetupHook` are used to vendor dependencies in the Python
`tokenizers` derivation. The `tokenizers` Python package is in the
`source/bindings/python` directory of the project's source archive. We
use `fetchCargoTarball` to retrieve the dependencies specified in
`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to
the `cargoDeps` attribute, which is used by `cargoSetupHook` to
configure Cargo.

```nix
{ fetchFromGitHub
, buildPythonPackage
, rustPlatform
, setuptools-rust
}:

buildPythonPackage rec {
pname = "tokenizers";
version = "0.10.0";

src = fetchFromGitHub {
owner = "huggingface";
repo = pname;
rev = "python-v${version}";
hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src sourceRoot;
name = "${pname}-${version}";
hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
};

sourceRoot = "source/bindings/python";

nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
cargoSetupHook
rust.cargo
rust.rustc
]);

# ...
}
```

In some projects, the Rust crate is not in the main source directory
of the projects. In such cases, the `cargoRoot` attribute can be used
to specify the crate's directory relative to `sourceRoot`. In the
following example, the crate is in `src/rust`, as specified in the
`cargoRoot` attribute. Note that we also need to specify the correct
path for `fetchCargoTarball`.

```nix

{ buildPythonPackage
, fetchPypi
, rustPlatform
, setuptools-rust
, openssl
}:

buildPythonPackage rec {
pname = "cryptography";
version = "3.4.2"; # Also update the hash in vectors.nix

src = fetchPypi {
inherit pname version;
sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
sourceRoot = "${pname}-${version}/${cargoRoot}";
name = "${pname}-${version}";
hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
};

cargoRoot = "src/rust";

# ...
}
```

## Compiling Rust crates using Nix instead of Cargo

### Simple operation
Expand Down
85 changes: 2 additions & 83 deletions pkgs/build-support/rust/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
, buildPackages
, cacert
, cargo
, diffutils
, cargoSetupHook
, fetchCargoTarball
, runCommandNoCC
, rustPlatform
Expand Down Expand Up @@ -71,19 +71,6 @@ let
# against the src fixed-output derivation to check consistency.
validateCargoDeps = !(cargoHash == "" && cargoSha256 == "");

# Some cargo builds include build hooks that modify their own vendor
# dependencies. This copies the vendor directory into the build tree and makes
# it writable. If we're using a tarball, the unpackFile hook already handles
# this for us automatically.
setupVendorDir = if cargoVendorDir == null
then (''
unpackFile "$cargoDeps"
cargoDepsCopy=$(stripHash $cargoDeps)
'')
else ''
cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
'';

targetIsJSON = lib.hasSuffix ".json" target;
useSysroot = targetIsJSON && !__internal_dontAddSysroot;

Expand All @@ -106,11 +93,6 @@ let
releaseDir = "target/${shortTarget}/${buildType}";
tmpDir = "${releaseDir}-tmp";

# Specify the stdenv's `diff` by abspath to ensure that the user's build
# inputs do not cause us to find the wrong `diff`.
# The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
diff = "${diffutils.nativeDrv or diffutils}/bin/diff";

in

# Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`.
Expand All @@ -124,7 +106,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u

patchRegistryDeps = ./patch-registry-deps;

nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ];
nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ];
buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads;

patches = cargoPatches ++ patches;
Expand All @@ -135,72 +117,9 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
postUnpack = ''
eval "$cargoDepsHook"

${setupVendorDir}

mkdir .cargo
config="$(pwd)/$cargoDepsCopy/.cargo/config";
if [[ ! -e $config ]]; then
config=${./fetchcargo-default-config.toml};
fi;
substitute $config .cargo/config \
--subst-var-by vendor "$(pwd)/$cargoDepsCopy"

cat >> .cargo/config <<'EOF'
[target."${rust.toRustTarget stdenv.buildPlatform}"]
"linker" = "${ccForBuild}"
${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
[target."${shortTarget}"]
"linker" = "${ccForHost}"
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
"rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
''}
''}
EOF

export RUST_LOG=${logLevel}
'' + (args.postUnpack or "");

# After unpacking and applying patches, check that the Cargo.lock matches our
# src package. Note that we do this after the patchPhase, because the
# patchPhase may create the Cargo.lock if upstream has not shipped one.
postPatch = (args.postPatch or "") + lib.optionalString validateCargoDeps ''
cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock
srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock

echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
if ! ${diff} $srcLockfile $cargoDepsLockfile; then

# If the diff failed, first double-check that the file exists, so we can
# give a friendlier error msg.
if ! [ -e $srcLockfile ]; then
echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
exit 1
fi

if ! [ -e $cargoDepsLockfile ]; then
echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
exit 1
fi

echo
echo "ERROR: cargoSha256 is out of date"
echo
echo "Cargo.lock is not the same in $cargoDepsCopy"
echo
echo "To fix the issue:"
echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
echo

exit 1
fi
'' + ''
unset cargoDepsCopy
'';

configurePhase = args.configurePhase or ''
runHook preConfigure
runHook postConfigure
Expand Down
84 changes: 84 additions & 0 deletions pkgs/build-support/rust/hooks/cargo-setup-hook.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
cargoSetupPostUnpackHook() {
echo "Executing cargoSetupPostUnpackHook"

# Some cargo builds include build hooks that modify their own vendor
# dependencies. This copies the vendor directory into the build tree and makes
# it writable. If we're using a tarball, the unpackFile hook already handles
# this for us automatically.
if [ -z $cargoVendorDir ]; then
unpackFile "$cargoDeps"
export cargoDepsCopy=$(stripHash $cargoDeps)
else
cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}"
fi

if [ ! -d .cargo ]; then
mkdir .cargo
fi

config="$(pwd)/$cargoDepsCopy/.cargo/config";
if [[ ! -e $config ]]; then
config=@defaultConfig@
fi;

tmp_config=$(mktemp)
substitute $config $tmp_config \
--subst-var-by vendor "$(pwd)/$cargoDepsCopy"
cat ${tmp_config} >> .cargo/config

cat >> .cargo/config <<'EOF'
@rustTarget@
EOF

echo "Finished cargoSetupPostUnpackHook"
}

# After unpacking and applying patches, check that the Cargo.lock matches our
# src package. Note that we do this after the patchPhase, because the
# patchPhase may create the Cargo.lock if upstream has not shipped one.
cargoSetupPostPatchHook() {
echo "Executing cargoSetupPostPatchHook"

cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock"
srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock"

echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
if ! @diff@ $srcLockfile $cargoDepsLockfile; then

# If the diff failed, first double-check that the file exists, so we can
# give a friendlier error msg.
if ! [ -e $srcLockfile ]; then
echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
exit 1
fi

if ! [ -e $cargoDepsLockfile ]; then
echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
exit 1
fi

echo
echo "ERROR: cargoSha256 is out of date"
echo
echo "Cargo.lock is not the same in $cargoDepsCopy"
echo
echo "To fix the issue:"
echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With fetchCargoTarball we need to pass another attribute. Not important for this PR though.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmpf, good point. We could change fetchCargoTarball to take cargoSha256/cargoHash instead, but that would make the argument non-canonical for a fetcher.

echo

exit 1
fi

unset cargoDepsCopy

echo "Finished cargoSetupPostPatchHook"
}

postUnpackHooks+=(cargoSetupPostUnpackHook)

if [ -z ${cargoVendorDir-} ]; then
postPatchHooks+=(cargoSetupPostPatchHook)
fi
49 changes: 49 additions & 0 deletions pkgs/build-support/rust/hooks/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{ buildPackages
, callPackage
, diffutils
, lib
, makeSetupHook
, rust
, stdenv
, target ? rust.toRustTargetSpec stdenv.hostPlatform
}:

let
targetIsJSON = lib.hasSuffix ".json" target;

# see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168
# the "${}" is needed to transform the path into a /nix/store path before baseNameOf
shortTarget = if targetIsJSON then
(lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
else target;
ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
in {
cargoSetupHook = callPackage ({ }:
makeSetupHook {
name = "cargo-setup-hook.sh";
deps = [ ];
substitutions = {
defaultConfig = ../fetchcargo-default-config.toml;

# Specify the stdenv's `diff` by abspath to ensure that the user's build
# inputs do not cause us to find the wrong `diff`.
# The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
diff = "${diffutils.nativeDrv or diffutils}/bin/diff";

# Target platform
rustTarget = ''
[target."${rust.toRustTarget stdenv.buildPlatform}"]
"linker" = "${ccForBuild}"
${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
[target."${shortTarget}"]
"linker" = "${ccForHost}"
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
"rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
''}
''}
'';
};
} ./cargo-setup-hook.sh) {};
}
5 changes: 4 additions & 1 deletion pkgs/development/compilers/rust/make-rust-platform.nix
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ rec {
};

buildRustPackage = callPackage ../../../build-support/rust {
inherit rustc cargo fetchCargoTarball;
inherit rustc cargo cargoSetupHook fetchCargoTarball;
};

rustcSrc = callPackage ./rust-src.nix {
Expand All @@ -22,4 +22,7 @@ rec {
rustLibSrc = callPackage ./rust-lib-src.nix {
inherit rustc;
};

# Hooks
inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook;
}
Loading