diff --git a/scratch/nested-extend.nix b/scratch/nested-extend.nix new file mode 100644 index 0000000..85c4dd7 --- /dev/null +++ b/scratch/nested-extend.nix @@ -0,0 +1,264 @@ +# ============================================================================ # +# +# +# +# ---------------------------------------------------------------------------- # + +let + +# ---------------------------------------------------------------------------- # + + nixpkgs = builtins.getFlake "nixpkgs"; + system = builtins.currentSystem; + pkgsFor = builtins.getAttr system nixpkgs.legacyPackages; + + +# ---------------------------------------------------------------------------- # + + # Causes `nodePackages' to use `node@20' + n20Overlay = final: prev: { nodejs = prev.nodejs-slim_20; }; + + # Add a package that requires `node@20' + myOverlay = final: prev: { + nodePackages = prev.nodePackages.extend ( nfinal: nprev: let + pkg-fun = { runCommand, nodejs, ... }: runCommand "bar" { + meta.nodeVersion = nodejs.version; + } '' + case "$( ${nodejs}/bin/node --version; )" in + v20.*) touch "$out"; ;; + v*) echo "v20 is required" >&2; exit 1; ;; + esac + ''; + in { + bar0 = final.callPackage pkg-fun {}; + bar1 = prev.callPackage pkg-fun {}; + bar2 = pkg-fun { inherit (prev) nodejs; inherit (final) runCommand; }; + bar3 = pkg-fun { inherit (final) nodejs; inherit (prev) runCommand; }; + bar4 = pkg-fun { inherit (prev) nodejs runCommand; }; + bar5 = pkg-fun { inherit (final) nodejs runCommand; }; + } ); + }; + + +# ---------------------------------------------------------------------------- # + + # Causes `nodePackages' to use `node@18' + n18Overlay = final: prev: { nodejs = prev.nodejs-18_x; }; + + # Add a package that requires `node@18' + friendOverlay = final: prev: { + nodePackages = prev.nodePackages.extend ( nfinal: nprev: let + pkg-fun = { runCommand, nodejs, ... }: runCommand "quux" { + meta.nodeVersion = nodejs.version; + } '' + case "$( ${nodejs}/bin/node --version; )" in + v18.*) touch "$out"; ;; + v*) echo "v18 is required" >&2; exit 1; ;; + esac + ''; + in { + quux0 = final.callPackage pkg-fun {}; + quux1 = prev.callPackage pkg-fun {}; + quux2 = pkg-fun { inherit (prev) nodejs; inherit (final) runCommand; }; + quux3 = pkg-fun { inherit (final) nodejs; inherit (prev) runCommand; }; + quux4 = pkg-fun { inherit (prev) nodejs runCommand; }; + quux5 = pkg-fun { inherit (final) nodejs runCommand; }; + } ); + }; + + +# ---------------------------------------------------------------------------- # + + extractNodeVersions = let + targets = [ + "bar0" "bar1" "bar2" "bar3" "bar4" "bar5" + "quux0" "quux1" "quux2" "quux3" "quux4" "quux5" + ]; + in pkgs: let + get = name: { + inherit name; + value = ( builtins.getAttr name pkgs.nodePackages ).meta.nodeVersion; + }; + in builtins.listToAttrs ( map get targets ); + + +# ---------------------------------------------------------------------------- # + + showNodeVersions = pkgs: let + extracted = extractNodeVersions pkgs; + showOne = name: version: let + spaces = if ( builtins.match "bar.*" name ) == null then " " else " "; + in name + ":" + spaces + "nodejs@" + version; + strs = builtins.mapAttrs showOne extracted; + in builtins.concatStringsSep "\n" ( builtins.attrValues strs ); + + +# ---------------------------------------------------------------------------- # + + chainOverlays = builtins.foldl' ( pkgs: pkgs.extend ) pkgsFor; + composeOverlays = overlays: + pkgsFor.extend ( nixpkgs.lib.composeManyExtensions overlays ); + +# ---------------------------------------------------------------------------- # + + pkgSets = { + chained0 = chainOverlays [n20Overlay myOverlay n18Overlay friendOverlay]; + chained1 = chainOverlays [n18Overlay friendOverlay n20Overlay myOverlay]; + chained2 = chainOverlays [friendOverlay n18Overlay myOverlay n20Overlay]; + chained3 = chainOverlays [myOverlay n20Overlay friendOverlay n18Overlay]; + composed0 = composeOverlays [n20Overlay myOverlay n18Overlay friendOverlay]; + composed1 = composeOverlays [n18Overlay friendOverlay n20Overlay myOverlay]; + composed2 = composeOverlays [friendOverlay n18Overlay myOverlay n20Overlay]; + composed3 = composeOverlays [myOverlay n20Overlay friendOverlay n18Overlay]; + }; + + versions = { + nix = builtins.mapAttrs ( _: extractNodeVersions ) pkgSets; + str = builtins.mapAttrs ( _: showNodeVersions ) pkgSets; + }; + + +# ---------------------------------------------------------------------------- # + +in { + + inherit (nixpkgs) lib; + inherit pkgsFor pkgSets versions; + + overlays = { + inherit n20Overlay n18Overlay myOverlay friendOverlay; + }; + util = { + inherit chainOverlays composeOverlays extractNodeVersions showNodeVersions; + }; + + show = str: builtins.trace ( "\n" + str ) null; + + report = let + reportOne = name: str: '' + * ${name} + - ${builtins.replaceStrings ["\n"] ["\n - "] str} + ''; + strs = builtins.mapAttrs reportOne versions.str; + in builtins.concatStringsSep "\n" ( builtins.attrValues strs ); + +} + +# ---------------------------------------------------------------------------- # +# +# Results +# ------- +# * chained0 +# - bar0: nodejs@18.16.0 +# - bar1: nodejs@18.16.0 +# - bar2: nodejs@20.2.0 +# - bar3: nodejs@18.16.0 +# - bar4: nodejs@20.2.0 +# - bar5: nodejs@18.16.0 +# - quux0: nodejs@18.16.0 +# - quux1: nodejs@18.16.0 +# - quux2: nodejs@18.16.0 +# - quux3: nodejs@18.16.0 +# - quux4: nodejs@18.16.0 +# - quux5: nodejs@18.16.0 +# +# * chained1 +# - bar0: nodejs@20.2.0 +# - bar1: nodejs@20.2.0 +# - bar2: nodejs@20.2.0 +# - bar3: nodejs@20.2.0 +# - bar4: nodejs@20.2.0 +# - bar5: nodejs@20.2.0 +# - quux0: nodejs@20.2.0 +# - quux1: nodejs@20.2.0 +# - quux2: nodejs@18.16.0 +# - quux3: nodejs@20.2.0 +# - quux4: nodejs@18.16.0 +# - quux5: nodejs@20.2.0 +# +# * chained2 +# - bar0: nodejs@20.2.0 +# - bar1: nodejs@20.2.0 +# - bar2: nodejs@18.16.0 +# - bar3: nodejs@20.2.0 +# - bar4: nodejs@18.16.0 +# - bar5: nodejs@20.2.0 +# - quux0: nodejs@20.2.0 +# - quux1: nodejs@20.2.0 +# - quux2: nodejs@18.16.0 +# - quux3: nodejs@20.2.0 +# - quux4: nodejs@18.16.0 +# - quux5: nodejs@20.2.0 +# +# * chained3 +# - bar0: nodejs@18.16.0 +# - bar1: nodejs@18.16.0 +# - bar2: nodejs@18.16.0 +# - bar3: nodejs@18.16.0 +# - bar4: nodejs@18.16.0 +# - bar5: nodejs@18.16.0 +# - quux0: nodejs@18.16.0 +# - quux1: nodejs@18.16.0 +# - quux2: nodejs@20.2.0 +# - quux3: nodejs@18.16.0 +# - quux4: nodejs@20.2.0 +# - quux5: nodejs@18.16.0 +# +# * composed0 +# - bar0: nodejs@18.16.0 +# - bar1: nodejs@18.16.0 +# - bar2: nodejs@20.2.0 +# - bar3: nodejs@18.16.0 +# - bar4: nodejs@20.2.0 +# - bar5: nodejs@18.16.0 +# - quux0: nodejs@18.16.0 +# - quux1: nodejs@18.16.0 +# - quux2: nodejs@18.16.0 +# - quux3: nodejs@18.16.0 +# - quux4: nodejs@18.16.0 +# - quux5: nodejs@18.16.0 +# +# * composed1 +# - bar0: nodejs@20.2.0 +# - bar1: nodejs@20.2.0 +# - bar2: nodejs@20.2.0 +# - bar3: nodejs@20.2.0 +# - bar4: nodejs@20.2.0 +# - bar5: nodejs@20.2.0 +# - quux0: nodejs@20.2.0 +# - quux1: nodejs@20.2.0 +# - quux2: nodejs@18.16.0 +# - quux3: nodejs@20.2.0 +# - quux4: nodejs@18.16.0 +# - quux5: nodejs@20.2.0 +# +# * composed2 +# - bar0: nodejs@20.2.0 +# - bar1: nodejs@20.2.0 +# - bar2: nodejs@18.16.0 +# - bar3: nodejs@20.2.0 +# - bar4: nodejs@18.16.0 +# - bar5: nodejs@20.2.0 +# - quux0: nodejs@20.2.0 +# - quux1: nodejs@20.2.0 +# - quux2: nodejs@18.16.0 +# - quux3: nodejs@20.2.0 +# - quux4: nodejs@18.16.0 +# - quux5: nodejs@20.2.0 +# +# * composed3 +# - bar0: nodejs@18.16.0 +# - bar1: nodejs@18.16.0 +# - bar2: nodejs@18.16.0 +# - bar3: nodejs@18.16.0 +# - bar4: nodejs@18.16.0 +# - bar5: nodejs@18.16.0 +# - quux0: nodejs@18.16.0 +# - quux1: nodejs@18.16.0 +# - quux2: nodejs@20.2.0 +# - quux3: nodejs@18.16.0 +# - quux4: nodejs@20.2.0 +# - quux5: nodejs@18.16.0 +# +# +# ============================================================================ # diff --git a/use-cases/deep-replace.md b/use-cases/deep-replace.md new file mode 100644 index 0000000..6448ccd --- /dev/null +++ b/use-cases/deep-replace.md @@ -0,0 +1,124 @@ +# Deeply Replacing Packages in Nixpkgs + +## General Information + +Replace all "instances" of a package across a dependency graph. +This may be replacing all usage in a package set, collection of package sets, +a collection of ad-hoc recipes, or a collection of flakes. + +A possible motivation for _deep replacement_ may be to ensure that a security +fix provided by a new release of a piece of software is used "everywhere" in +the dependency graph. + + +## Concrete Examples + +While the precise organization of packages will effect the complexity and +effort required to perform _deep replacement_, in general we say that this +is accomplished using helper functions such as `extend`, `overrideScope'`, and +`appendOverlays`, as well as the configuration field `overlays`. + +### Simple Overlay + +```nix +pkgs.extend ( final: prev: { foo = final.callPackage ./my-pkgs/foo {}; } ) +``` + + + +### Nested Overlays + +Nested overlays, "chained" vs. "composed" overlays, as and when to use `prev` +and `final` are common sources of confusion. +This can be particularly problematic when consuming overlays defined externally. + +There is an extended example [here](../scratch/nested-extend.nix) that shows a +variety of common pitfalls, but the snippet below shows an abbreviated case. +Here the use of `nodejs` is contrived, what's important is that `nodejs` is +used to populate a nested scope `nodePackages`. +For the purposes of this example, we'll imagine the user is unaware of how +`nodePackages` attrsets should be used and naively use `nodePackages` +to add their own package definitions. +The core of this issue is that we attempt to merge an external overlay which +uses `nodejs@18` with an overlay using `nodejs@20`. +The user's goal here is really to use `nodejs@20` for the packages they define +and for their dependencies, but doing so inadvertently modifies executables +defined in our external overlay. + +External Flake: +```nix +{ + outputs = { nixpkgs, ... }: let + # We must split these into two overlays so `prev` holds the correct `nodejs` + overlays.node18 = final: prev: { nodejs = prev.nodejs-18_x; }; + overlays.theirPkgs = final: prev: { + nodePackages = prev.nodePackages.extend ( nfinal: nprev: { + # Requires `nodejs@18' to avoid runtime errors. + someExecutable = nfinal.callPackage ./. {}; + } ); + }; + overlays.default = + nixpkgs.lib.composeExtensions overlays.node18 overlays.theirPkgs; + in { inherit overlays; } +} +``` + +Our Flake: +``` +{ + outputs = { nixpkgs, other, ... }: let + overlays.deps = other.overlays.default; + overlays.node20 = final: prev: { nodejs = prev.nodejs-20_x; }; + overlays.myPkgs = final: prev: { + nodePackages = prev.nodePackages.extend ( nfinal: nprev: { + # Doesn't use any node modules from previous overlay, but requires + # `nodejs@20' to avoid runtime errors. + myModule = nfinal.callPackage ./module {}; + } ); + # Uses `someExecutable' from previous overlay. + someTool = final.callPackage ./tool {}; + }; + overlays.default = nixpkgs.lib.composeManyExtensions [ + overlays.deps overlays.node20 overlays.myPkgs + ]; + in { + inherit overlays; + packages.x86_64-linux = let + pkgsFor = nixpkgs.legacyPackages.x86_64-linux.extend overlays.default; + in { inherit (pkgsFor) nodePackages someTool; } + # packages. = ...; + } +} +``` + +In this example the user will encounter runtime crashes in `myTool` caused by +accidentally overriding `nodejs = nodejs-20_x;` +in `nodePackages.someExecutable` defined externally. +While we certainly have ways to avoid this category of issue, the process seen +above is already a challenge to understand - ideally a more intuitive pattern +could be offered to users. + + +## Current Problems + +With this approach we have three main sources of complexity, none of which +truly prevent a user from accomplishing their goal, but we might suffice to +say that it may be worthwhile to provide a more straightforward mechanism +for handling this use case. + +1. [github:NixOS/nixpkgs://lib/customisation.nix](https://github.com/NixOS/nixpkgs/blob/master/lib/customisation.nix) +routines aren't intuitively understood by many users. + - `self`/`super` and `final`/`prev` are difficult for new users to understand. + - Easy to accidentally trigger infinite recursion. + - Some packages are not _truly_ overridable, which advanced users will only + discover after a lengthy debugging session. + +2. Nested scopes are difficult to locate, and the relationship between + parent scopes and child scopes is opaque to users. + - See [Nested Overlays](#Nested-Ex) example. + +3. With ad-hoc recipes and flakes there isn't standardized usage of + `overlays` that allow deep overriding of packages transitively. + - Improved guidance on the use of `overlays` and `follows` in `flakes` + could help a bit here. +