Skip to content

Commit

Permalink
Revert "Overhaul Workplace Shell & Overrides"
Browse files Browse the repository at this point in the history
This reverts commit 2ae2f57.
  • Loading branch information
proski committed Feb 15, 2024
1 parent 93092cd commit 49f23eb
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 172 deletions.
2 changes: 1 addition & 1 deletion overlay/make-package-set/internal.nix
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ lib.fix' (self:

in packageFunWith { mkRustCrate = mkRustCrate'; buildRustPackages = buildRustPackages'; } // {
inherit rustPackages callPackage pkgs rustToolchain noBuild;
workspaceShell = workspaceShell { inherit pkgs rustPackages rustToolchain; };
workspaceShell = workspaceShell { inherit pkgs noBuild rustToolchain; };
mkRustCrate = mkRustCrate';
buildRustPackages = buildRustPackages';
__splicedPackages = defaultScope;
Expand Down
4 changes: 0 additions & 4 deletions overlay/mkcrate.nix
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,6 @@ let
inherit src version meta NIX_DEBUG;
name = "crate-${name}-${version}${optionalString (compileMode != "build") "-${compileMode}"}";

# Adding libiconv is a convenience hack. It really isn't needed by every
# derivation and should instead be added / propagated where appropriate, but
# until someone decides to investigate the actual dependencies, it remains
# here instead of in overrides.
buildInputs = runtimeDependencies ++ lib.optionals stdenv.hostPlatform.isDarwin [ pkgs.libiconv ];
propagatedBuildInputs = lib.unique (concatMap (drv: drv.propagatedBuildInputs) runtimeDependencies);
nativeBuildInputs = [ rustToolchain ] ++ buildtimeDependencies;
Expand Down
118 changes: 46 additions & 72 deletions overlay/overrides.nix
Original file line number Diff line number Diff line change
@@ -1,18 +1,6 @@
{ rustLib, lib, pkgs, buildPackages }:

let
inherit (rustLib) makeOverride nullOverride;

# The bindings in this let expression are used below in the recursive set to
# create overrides. They are not themselves overrides. See the list of `all`
# overrides and their definitions below.

# For example, openssl builds to separate directories in nixpkgs while the
# Rust crate expects all of the output to be in one directory. We use a
# symlink join to create an output of the sum of two of openssl's normal
# outputs. This is one example of a nixpkg requiring some slight finesse to
# be used as a buildInput for a Rust crate derivation.

envize = s: builtins.replaceStrings ["-"] ["_"] (lib.toUpper s);

patchOpenssl = pkgs:
Expand Down Expand Up @@ -56,8 +44,25 @@ let
libkrb5 = pkgs.libkrb5.override { inherit openssl; };
};

propagateEnv = name: envs: buildPackages.stdenv.mkDerivation {
name = "${name}-propagate-env";
setupHook = buildPackages.writeText "exports.sh" ''
${name}-setup-env() {
${lib.concatMapStringsSep
"\n"
({ name, value }: "export ${name}=${lib.escapeShellArg value}")
envs}
}
addEnvHooks "$hostOffset" ${name}-setup-env
'';
phases = "installPhase fixupPhase";
installPhase = "mkdir -p $out";
preferLocalBuild = true;
allowSubstitutes = false;
};

in rec {
patches = { inherit patchOpenssl patchCurl patchPostgresql joinOpenssl;};
patches = { inherit patchOpenssl patchCurl patchPostgresql joinOpenssl propagateEnv; };

# Don't forget to add new overrides here.
all = [
Expand Down Expand Up @@ -88,7 +93,6 @@ in rec {
overrideArgs = old: { rustcLinkFlags = old.rustcLinkFlags or [ ] ++ [ "--cap-lints" "warn" ]; };
};

# Every crate that depends on the cc crate (usually build scripts) will have the xcbuild
cc = if pkgs.stdenv.hostPlatform.isDarwin
then makeOverride {
name = "cc";
Expand All @@ -103,9 +107,7 @@ in rec {
curl-sys = makeOverride {
name = "curl-sys";
overrideAttrs = drv: {
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
(patchCurl pkgs)
];
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ (patchCurl pkgs) ];
};
};

Expand Down Expand Up @@ -134,7 +136,7 @@ in rec {
libdbus-sys = pkgs.rustBuilder.rustLib.makeOverride {
name = "libdbus-sys";
overrideAttrs = drv: {
buildInputs = drv.buildInputs or [ ] ++ [
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
pkgs.dbus
];
};
Expand All @@ -143,7 +145,7 @@ in rec {
libudev-sys = pkgs.rustBuilder.rustLib.makeOverride {
name = "libudev-sys";
overrideAttrs = drv: {
buildInputs = drv.buildInputs or [ ] ++ [
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
pkgs.udev
];
buildPhase = ''
Expand All @@ -162,8 +164,6 @@ in rec {
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
pkgs.darwin.apple_sdk.frameworks.Security
pkgs.darwin.apple_sdk.frameworks.CoreFoundation
];
buildInputs = drv.buildInputs or [ ] ++ [
pkgs.libgit2
];
preferLocalBuild = true;
Expand All @@ -175,102 +175,76 @@ in rec {
libssh2-sys = makeOverride {
name = "libssh2-sys";
overrideAttrs = drv: {
buildInputs = drv.buildInputs or [ ] ++ [ pkgs.openssl.dev pkgs.zlib.dev ];
propagatedNativeBuildInputs = drv.propagatedNativeBuildInputs or [ ] ++ [ pkgs.openssl.dev pkgs.zlib.dev ];
};
};

libsqlite3-sys = pkgs.rustBuilder.rustLib.makeOverride {
name = "libsqlite3-sys";
overrideAttrs = drv: {
buildInputs = drv.buildInputs or [ ] ++ [ pkgs.sqlite ];
propagatedNativeBuildInputs = drv.propagatedNativeBuildInputs or [ ] ++ [ pkgs.sqlite ];
};
};

openssl-sys = makeOverride {
name = "openssl-sys";
overrideAttrs = drv: {
# The setup hook will set the variables both for building openssl-sys and
# in dependent derivations. This mechanism will also set the variable
# inside our development shell. Because the setupHook does not add the
# joinOpenssl derivation as a dependnecy, we have to include it in
# nativeBuildInputs as well or the variable will point to a path not
# visible to the derivation at build time.
buildInputs = drv.buildInputs or [ ] ++ [(joinOpenssl (patchOpenssl pkgs.buildPackages))];

shellHook = drv.shellHook or "" + ''
export ${envize (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.buildPlatform)}_OPENSSL_DIR=${lib.escapeShellArg (joinOpenssl (patchOpenssl pkgs.buildPackages))}
export ${envize (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.hostPlatform)}_OPENSSL_DIR=${lib.escapeShellArg (joinOpenssl (patchOpenssl pkgs))}
export OPENSSL_NO_VENDOR=1 # fixed 0.9.60
export RUSTFLAGS="''${RUSTFLAGS:-} --cfg ossl111 --cfg ossl110 --cfg ossl101"
'';

# setupHook is also a means of injecting the build environment for a dependency
# setupHook = buildPackages.writeText "openssl-sys-setup-env.sh" ''
# openssl-sys-setup-env() {
# export ${envize (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.buildPlatform)}_OPENSSL_DIR=${lib.escapeShellArg (joinOpenssl (patchOpenssl pkgs.buildPackages))}
# export ${envize (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.hostPlatform)}_OPENSSL_DIR=${lib.escapeShellArg (joinOpenssl (patchOpenssl pkgs))}
# export OPENSSL_NO_VENDOR=1 # fixed 0.9.60
# export RUSTFLAGS="''${RUSTFLAGS:-} --cfg ossl111 --cfg ossl110 --cfg ossl101"
# }
# addEnvHooks "$hostOffset" openssl-sys-setup-env
# '';
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
(propagateEnv "openssl-sys" [
{ name = "RUSTFLAGS"; value = "--cfg ossl111 --cfg ossl110 --cfg ossl101";}
{ name = "${envize (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.buildPlatform)}_OPENSSL_DIR"; value = joinOpenssl (patchOpenssl pkgs.buildPackages); }
{ name = "${envize (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.hostPlatform)}_OPENSSL_DIR"; value = joinOpenssl (patchOpenssl pkgs); }
{ name = "OPENSSL_NO_VENDOR"; value = "1";} # fixed 0.9.60
])
];
};
};

pkg-config = makeOverride {
name = "pkg-config";
overrideAttrs = drv: {
# Every crate that depends on the pkg-config crate also gets pkg-config and this environment
propagatedNativeBuildInputs = drv.propagatedNativeBuildInputs or [ ] ++ [
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
pkgs.pkg-config
(propagateEnv "pkg-config" [
{ name = "PKG_CONFIG_ALLOW_CROSS"; value = "1"; }
])
];
shellHook = drv.shellHook or "" + ''
export PGK_CONFIG_ALLOW_CROSS=1
'';
};
};

pq-sys =
let
binEcho = s: "${pkgs.buildPackages.writeShellScriptBin "bin-echo" "echo ${s}"}/bin/bin-echo";
fake_pg_config = binEcho "${(patchPostgresql pkgs.buildPackages).lib}/lib";
in
makeOverride {
name = "pq-sys";
overrideAttrs = drv: {
# We can't use the host `pg_config` here, as it might not run on build platform. `pq-sys` only needs
# to know the `lib` directory for `libpq`, so just create a fake binary that gives it exactly that.
nativeBuildInputs = drv.nativeBuildInputs or [ ] ++ [
fake_pg_config
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
(propagateEnv "pq-sys" [
{ name = "PG_CONFIG_${envize pkgs.stdenv.buildPlatform.config}"; value = binEcho "${(patchPostgresql pkgs.buildPackages).lib}/lib"; }
{ name = "PG_CONFIG_${envize pkgs.stdenv.hostPlatform.config}"; value = binEcho "${(patchPostgresql pkgs).lib}/lib"; }
])
];
shellHook = drv.shellHook + ''
PG_CONFIG_${envize pkgs.stdenv.buildPlatform.config}="${fake_pg_config}"
'';
};
};

# Note that protobuff is from buildPackages and runs at the crate's
# build-time, so it's a nativeBuildInput. Every crate that depends on
# prost-build might need protoc at runtime, so it's propagated.
prost-build = makeOverride {
name = "prost-build";
overrideAttrs = drv: {
setupHook = buildPackages.writeText "prost-build-setup-env.sh" ''
prost-build-setup-env () {
PROTOC="${pkgs.buildPackages.buildPackages.protobuf}/bin/protoc"
}
addEnvHooks "$hostOffset" prost-build-setup-env
'';
propagatedNativeBuildInputs = drv.propagatedNativeBuildInputs or [ ] ++ [
pkgs.buildPackages.buildPackages.protobuf
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
(propagateEnv "prost-build" [
{ name = "PROTOC"; value = "${pkgs.buildPackages.buildPackages.protobuf}/bin/protoc"; }
])
];
};
};

protoc = makeOverride {
name = "protoc";
overrideAttrs = drv: {
propagatedNativeBuildInputs = drv.propagatedNativeBuildInputs or [ ] ++ [ pkgs.buildPackages.buildPackages.protobuf ];
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ pkgs.buildPackages.buildPackages.protobuf ];
};
};

Expand Down Expand Up @@ -323,7 +297,7 @@ in rec {
zmq-sys = makeOverride {
name = "zmq-sys";
overrideAttrs = drv: {
buildInputs = drv.buildInputs or [ ] ++ [ pkgs.zeromq ];
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ pkgs.zeromq ];
};
};
}
108 changes: 13 additions & 95 deletions overlay/workspace-shell.nix
Original file line number Diff line number Diff line change
@@ -1,98 +1,16 @@
{ pkgs, rustPackages, rustToolchain }:
# The resulting function is a decoration of mkShell. Usually mkShell is only
# given an inputsFrom that is a single package or packages without overlap in
# dependencies. However, we seek to provide a shell that is complete for every
# rust crate in the entire workspace. However, if we naively pass all rust
# crates to inputsFrom, many dependencies will be duplicated via propagation or
# multiple inclusion. Therefore, instead create the dependency sets from the
# rust crates are collected and de-duplicated before passing them to the normal
# mkShell. This is also done for the shellHook, just slightly differently. The
# source is really similar to nixpkgs mkShell that it decorates, so study one
# and understand both.
{ name ? "nix-shell"
, # a list of packages to add to the shell environment
packages ? [ ]
, # propagate all the inputs from the given derivations
inputsFrom ? [ ]
, buildInputs ? [ ]
, nativeBuildInputs ? [ ]
, propagatedBuildInputs ? [ ]
, propagatedNativeBuildInputs ? [ ]
, ...
}@attrs:
let
lib = pkgs.lib;

# mkShell in the end will pass through arguments that it doesn't explicitly
# handle onward to mkDerivation. Arguments removed at this point will be
# consumed and propagated by this function.
rest = builtins.removeAttrs attrs [
"buildInputs"
"nativeBuildInputs"
"propagatedBuildInputs"
"propagatedNativeBuildInputs"
"shellHook"
];

# The crate functions from which we will gather the inputs must be called to
# yield finished crate derivations. It is important to note that they will be
# called with their default arguments. Augmentation of the crates which may
# affect their dependencies in the user's flake will result in inconsistency,
# so if such behavior exists or is added by the user, a mechanism must be
# introduced to propagate these effects on dependencies into the shell.

# TODO note that if package set is created from nixpkgs for another platform,
# it will be inappropriate for creating a workspace shell. Flakes written for
# cross compilation but still get their workspaceShell from a package set for
# the build platform.

# TODO This is fragile because any path in the entire set that is not a crate
# function will cause failure. Recent changes to overlay/make-package-set or
# overlay/default, even changes to Nix itself could add a path. Replace this
# with some construct with no possibility of non-crate contamination.
crateFunctions = builtins.removeAttrs rustPackages
["workspace"
"workspaceShell"
"cargo2nixVersion"
"rustToolchain"
"rustPackages"
"pkgs"
"noBuild"
"mkRustCrate"
"callPackage"
"buildRustPackages"
"__unfix__"
"__splicedPackages"];

# Note that out paths must match between the crate and the crates dependencies
# (crates depend on crate.out) or else you will get multiple inclusion and
# crates themselves in the shell depencnedies. Cargo can build rust deps, so
# they are not needed in the development shell, nor will they be used by cargo
# outside of nix builds.
crates = map (pkg: (pkg { }).out) (pkgs.lib.collect builtins.isFunction crateFunctions);

# This function will extract the attr "name" from all crates, augment this
# list with the "name" from @attrs, remove explicit overlap with inputsFrom,
# and finally de-duplicate with unique.
mergeCrateInputs = name:
(lib.unique
(lib.subtractLists (packages ++ inputsFrom ++ crates)
((attrs.${name} or [ ]) ++ (lib.flatten (lib.catAttrs name crates)))));

in pkgs.mkShell (rest // {

# TODO investigate if cacert is needed or had been omitted in previous implementation
buildInputs = mergeCrateInputs "buildInputs";
nativeBuildInputs = (mergeCrateInputs "nativeBuildInputs") ++ [ rustToolchain ] ++ (with pkgs; [cacert]);
propagatedBuildInputs = mergeCrateInputs "propagatedBuildInputs";
propagatedNativeBuildInputs = mergeCrateInputs "propagatedNativeBuildInputs";

# Create a composite shellHook from the user passed shellHook and the rust
# crates' shellHooks. This hook will be merged by mkShell with the shellHooks
# in inputsFrom
shellHook = lib.concatStringsSep "\n" (lib.unique (lib.catAttrs "shellHook"
(lib.reverseList crates ++ [ attrs ])));

{ pkgs, noBuild, rustToolchain }:
args@{
inputsFrom ? [],
nativeBuildInputs ? [],
...
}:

pkgs.mkShell (args // {
# `noBuild` is a special crate set used to create a development shell
# containing all native dependencies provided by the overrides above.
# `cargo build` within the shell should just work.
inputsFrom = (pkgs.lib.mapAttrsToList (_: pkg: pkg { }) noBuild.workspace) ++ inputsFrom;
nativeBuildInputs = [ rustToolchain ] ++ (with pkgs; [cacert]) ++ nativeBuildInputs;
# Configures tools like Rust Analyzer to locate the correct rust-src
RUST_SRC_PATH = "${rustToolchain}/lib/rustlib/src/rust/library";
})

0 comments on commit 49f23eb

Please sign in to comment.