buildRustPackage: factor out setting up cargoBuildHook, add maturinBuildHook

doc/languages-frameworks/rust.section.md

@@ -248,18 +248,65 @@ hooks that can be used to integrate Cargo in non-Rust packages.
 Since network access is not allowed in sandboxed builds, Rust crate
 dependencies need to be retrieved using a fetcher. `rustPlatform`
 provides the `fetchCargoTarball` fetcher, which vendors all
-dependencies of a crate. This fetcher can be used jointly with
-`cargoSetupHook` to vendor dependencies in derivations that do not use
-`buildRustPackage`.
-
-In the following partial example, `fetchCargoTarball` and
-`cargoSetupHook` are used to vendor dependencies in the Python
-`tokenizers` derivation. The `tokenizers` Python package is in the
-`source/bindings/python` directory of the project's source archive. We
-use `fetchCargoTarball` to retrieve the dependencies specified in
-`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to
-the `cargoDeps` attribute, which is used by `cargoSetupHook` to
-configure Cargo.
+dependencies of a crate. For example, given a source path `src`
+containing `Cargo.toml` and `Cargo.lock`, `fetchCargoTarball`
+can be used as follows:
+
+```nix
+cargoDeps = rustPlatform.fetchCargoTarball {
+  inherit src;
+  hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
+};
+```
+
+The `src` attribute is required, as well as a hash specified through
+one of the `sha256` or `hash` attributes. The following optional
+attributes can also be used:
+
+* `name`: the name that is used for the dependencies tarball.  If
+  `name` is not specified, then the name `cargo-deps` will be used.
+* `sourceRoot`: when the `Cargo.lock`/`Cargo.toml` are in a
+  subdirectory, `sourceRoot` specifies the relative path to these
+  files.
+* `patches`: patches to apply before vendoring. This is useful when
+  the `Cargo.lock`/`Cargo.toml` files need to be patched before
+  vendoring.
+
+### Hooks
+
+`rustPlatform` provides the following hooks to automate Cargo builds:
+
+* `cargoSetupHook`: configure Cargo to use depenencies vendored
+  through `fetchCargoTarball`. This hook uses the `cargoDeps`
+  environment variable to find the vendored dependencies. If a project
+  already vendors its dependencies, the variable `cargoVendorDir` can
+  be used instead. When the `Cargo.toml`/`Cargo.lock` files are not in
+  `sourceRoot`, then the optional `cargoRoot` is used to specify the
+  Cargo root directory relative to `sourceRoot`.
+* `cargoBuildHook`: use Cargo to build a crate. If the crate to be
+  built is a crate in e.g. a Cargo workspace, the relative path to the
+  crate to build can be set through the optional `buildAndTestSubdir`
+  environment variable. Additional Cargo build flags can be passed
+  through `cargoBuildFlags`.
+* `maturinBuildHook`: use [Maturin](https://github.com/PyO3/maturin)
+  to build a Python wheel. Similar to `cargoBuildHook`, the optional
+  variable `buildAndTestSubdir` can be used to build a crate in a
+  Cargo workspace. Additional maturin flags can be passed through
+  `maturinBuildFlags`.
+
+### Examples
+
+#### Python package using `setuptools-rust`
+
+For Python packages using `setuptools-rust`, you can use
+`fetchCargoTarball` and `cargoSetupHook` to retrieve and set up Cargo
+dependencies. The build itself is then performed by
+`buildPythonPackage`.
+
+The following example outlines how the `tokenizers` Python package is
+built. Since the Python package is in the `source/bindings/python`
+directory of the *tokenizers* project's source archive, we use
+`sourceRoot` to point the tooling to this directory:
 
 ```nix
 { fetchFromGitHub
@@ -297,9 +344,9 @@ buildPythonPackage rec {
 }
 ```
 
-In some projects, the Rust crate is not in the main source directory
-of the projects. In such cases, the `cargoRoot` attribute can be used
-to specify the crate's directory relative to `sourceRoot`. In the
+In some projects, the Rust crate is not in the main Python source
+directory.  In such cases, the `cargoRoot` attribute can be used to
+specify the crate's directory relative to `sourceRoot`. In the
 following example, the crate is in `src/rust`, as specified in the
 `cargoRoot` attribute. Note that we also need to specify the correct
 path for `fetchCargoTarball`.
@@ -335,6 +382,47 @@ buildPythonPackage rec {
 }
 ```
 
+#### Python package using `maturin`
+
+Python packages that use [Maturin](https://github.com/PyO3/maturin)
+can be built with `fetchCargoTarball`, `cargoSetupHook`, and
+`maturinBuildHook`. For example, the following (partial) derivation
+builds the `retworkx` Python package. `fetchCargoTarball` and
+`cargoSetupHook` are used to fetch and set up the crate dependencies.
+`maturinBuildHook` is used to perform the build.
+
+```nix
+{ lib
+, buildPythonPackage
+, rustPlatform
+, fetchFromGitHub
+}:
+
+buildPythonPackage rec {
+  pname = "retworkx";
+  version = "0.6.0";
+
+  src = fetchFromGitHub {
+    owner = "Qiskit";
+    repo = "retworkx";
+    rev = version;
+    sha256 = "11n30ldg3y3y6qxg3hbj837pnbwjkqw3nxq6frds647mmmprrd20";
+  };
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    name = "${pname}-${version}";
+    hash = "sha256-heOBK8qi2nuc/Ib+I/vLzZ1fUUD/G/KTw9d7M4Hz5O0=";
+  };
+
+  format = "pyproject";
+
+  nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ];
+
+  # ...
+}
+```
+
 ## Compiling Rust crates using Nix instead of Cargo
 
 ### Simple operation

pkgs/build-support/rust/default.nix

@@ -3,6 +3,7 @@
 , buildPackages
 , cacert
 , cargo
+, cargoBuildHook
 , cargoSetupHook
 , fetchCargoTarball
 , runCommandNoCC
@@ -37,7 +38,6 @@
 , cargoBuildFlags ? []
 , buildType ? "release"
 , meta ? {}
-, target ? rust.toRustTargetSpec stdenv.hostPlatform
 , cargoVendorDir ? null
 , checkType ? buildType
 , depsExtraArgs ? {}
@@ -71,6 +71,7 @@ let
   # against the src fixed-output derivation to check consistency.
   validateCargoDeps = !(cargoHash == "" && cargoSha256 == "");
 
+  target = rust.toRustTargetSpec stdenv.hostPlatform;
   targetIsJSON = lib.hasSuffix ".json" target;
   useSysroot = targetIsJSON && !__internal_dontAddSysroot;
 
@@ -86,10 +87,6 @@ let
     originalCargoToml = src + /Cargo.toml; # profile info is later extracted
   };
 
-  ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
-  cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
-  ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
-  cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
   releaseDir = "target/${shortTarget}/${buildType}";
   tmpDir = "${releaseDir}-tmp";
 
@@ -102,11 +99,17 @@ assert useSysroot -> !(args.doCheck or true);
 stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs useSysroot {
   RUSTFLAGS = "--sysroot ${sysroot} " + (args.RUSTFLAGS or "");
 } // {
-  inherit cargoDeps;
+  inherit buildAndTestSubdir cargoDeps releaseDir tmpDir;
+
+  cargoBuildFlags = lib.concatStringsSep " " cargoBuildFlags;
+
+  cargoBuildType = "--${buildType}";
 
   patchRegistryDeps = ./patch-registry-deps;
 
-  nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ];
+  nativeBuildInputs = nativeBuildInputs ++
+    [ cacert git cargo cargoBuildHook cargoSetupHook rustc ];
+
   buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads;
 
   patches = cargoPatches ++ patches;
@@ -125,38 +128,6 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
     runHook postConfigure
   '';
 
-  buildPhase = with builtins; args.buildPhase or ''
-    ${lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"}
-    runHook preBuild
-
-    (
-    set -x
-    env \
-      "CC_${rust.toRustTarget stdenv.buildPlatform}"="${ccForBuild}" \
-      "CXX_${rust.toRustTarget stdenv.buildPlatform}"="${cxxForBuild}" \
-      "CC_${rust.toRustTarget stdenv.hostPlatform}"="${ccForHost}" \
-      "CXX_${rust.toRustTarget stdenv.hostPlatform}"="${cxxForHost}" \
-      cargo build -j $NIX_BUILD_CORES \
-        ${lib.optionalString (buildType == "release") "--release"} \
-        --target ${target} \
-        --frozen ${concatStringsSep " " cargoBuildFlags}
-    )
-
-    runHook postBuild
-
-    ${lib.optionalString (buildAndTestSubdir != null) "popd"}
-
-    # This needs to be done after postBuild: packages like `cargo` do a pushd/popd in
-    # the pre/postBuild-hooks that need to be taken into account before gathering
-    # all binaries to install.
-    mkdir -p $tmpDir
-    cp -r $releaseDir/* $tmpDir/
-    bins=$(find $tmpDir \
-      -maxdepth 1 \
-      -type f \
-      -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \))
-  '';
-
   checkPhase = args.checkPhase or (let
     argstr = "${lib.optionalString (checkType == "release") "--release"} --target ${target} --frozen";
     threads = if cargoParallelTestThreads then "$NIX_BUILD_CORES" else "1";
@@ -173,11 +144,19 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
 
   strictDeps = true;
 
-  inherit releaseDir tmpDir;
-
   installPhase = args.installPhase or ''
     runHook preInstall
 
+    # This needs to be done after postBuild: packages like `cargo` do a pushd/popd in
+    # the pre/postBuild-hooks that need to be taken into account before gathering
+    # all binaries to install.
+    mkdir -p $tmpDir
+    cp -r $releaseDir/* $tmpDir/
+    bins=$(find $tmpDir \
+      -maxdepth 1 \
+      -type f \
+      -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \))
+
     # rename the output dir to a architecture independent one
     mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep '${tmpDir}$')
     for target in "''${targets[@]}"; do

pkgs/build-support/rust/fetchCargoTarball.nix

@@ -21,7 +21,7 @@ in
 , src ? null
 , srcs ? []
 , patches ? []
-, sourceRoot
+, sourceRoot ? ""
 , hash ? ""
 , sha256 ? ""
 , cargoUpdateHook ? ""

pkgs/build-support/rust/hooks/cargo-build-hook.sh

@@ -0,0 +1,33 @@
+cargoBuildHook() {
+    echo "Executing cargoBuildHook"
+
+    runHook preBuild
+
+    if [ ! -z "${buildAndTestSubdir-}" ]; then
+        pushd "${buildAndTestSubdir}"
+    fi
+
+    (
+    set -x
+    env \
+      "CC_@rustBuildPlatform@=@ccForBuild@" \
+      "CXX_@rustBuildPlatform@=@cxxForBuild@" \
+      "CC_@rustTargetPlatform@=@ccForHost@" \
+      "CXX_@rustTargetPlatform@=@cxxForHost@" \
+      cargo build -j $NIX_BUILD_CORES \
+        --target @rustTargetPlatformSpec@ \
+        --frozen \
+        ${cargoBuildType} \
+        ${cargoBuildFlags}
+    )
+
+    runHook postBuild
+
+    if [ ! -z "${buildAndTestSubdir-}" ]; then
+        popd
+    fi
+
+    echo "Finished cargoBuildHook"
+}
+
+buildPhase=cargoBuildHook

pkgs/build-support/rust/hooks/default.nix

@@ -1,8 +1,10 @@
 { buildPackages
 , callPackage
+, cargo
 , diffutils
 , lib
 , makeSetupHook
+, maturin
 , rust
 , stdenv
 , target ? rust.toRustTargetSpec stdenv.hostPlatform
@@ -16,9 +18,24 @@ let
   shortTarget = if targetIsJSON then
       (lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
     else target;
-  ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
-  ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
+  ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
+  cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
+  ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
+  cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
+  rustBuildPlatform = rust.toRustTarget stdenv.buildPlatform;
+  rustTargetPlatform = rust.toRustTarget stdenv.hostPlatform;
+  rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform;
 in {
+  cargoBuildHook = callPackage ({ }:
+    makeSetupHook {
+      name = "cargo-build-hook.sh";
+      deps = [ cargo ];
+      substitutions = {
+        inherit ccForBuild ccForHost cxxForBuild cxxForHost
+          rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec;
+      };
+    } ./cargo-build-hook.sh) {};
+
   cargoSetupHook = callPackage ({ }:
     makeSetupHook {
       name = "cargo-setup-hook.sh";
@@ -46,4 +63,14 @@ in {
         '';
       };
     } ./cargo-setup-hook.sh) {};
+
+  maturinBuildHook = callPackage ({ }:
+    makeSetupHook {
+      name = "maturin-build-hook.sh";
+      deps = [ cargo maturin ];
+      substitutions = {
+        inherit ccForBuild ccForHost cxxForBuild cxxForHost
+          rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec;
+      };
+    } ./maturin-build-hook.sh) {};
 }

pkgs/build-support/rust/hooks/maturin-build-hook.sh

@@ -0,0 +1,39 @@
+maturinBuildHook() {
+    echo "Executing maturinBuildHook"
+
+    runHook preBuild
+
+    if [ ! -z "${buildAndTestSubdir-}" ]; then
+        pushd "${buildAndTestSubdir}"
+    fi
+
+    (
+    set -x
+    env \
+      "CC_@rustBuildPlatform@=@ccForBuild@" \
+      "CXX_@rustBuildPlatform@=@cxxForBuild@" \
+      "CC_@rustTargetPlatform@=@ccForHost@" \
+      "CXX_@rustTargetPlatform@=@cxxForHost@" \
+      maturin build \
+        --cargo-extra-args="-j $NIX_BUILD_CORES --frozen" \
+        --target @rustTargetPlatformSpec@ \
+        --manylinux off \
+        --strip \
+        --release \
+        ${maturinBuildFlags-}
+    )
+
+    runHook postBuild
+
+    if [ ! -z "${buildAndTestSubdir-}" ]; then
+        popd
+    fi
+
+    # Move the wheel to dist/ so that regular Python tooling can find it.
+    mkdir -p dist
+    mv target/wheels/*.whl dist/
+
+    echo "Finished maturinBuildHook"
+}
+
+buildPhase=maturinBuildHook