use-case: deeply replace packages

scratch/nested-extend.nix

@@ -0,0 +1,264 @@
+# ============================================================================ #
+#
+#
+#
+# ---------------------------------------------------------------------------- #
+
+let
+
+# ---------------------------------------------------------------------------- #
+
+  nixpkgs = builtins.getFlake "nixpkgs";
+  system  = builtins.currentSystem;
+  pkgsFor = builtins.getAttr system nixpkgs.legacyPackages;
+
+
+# ---------------------------------------------------------------------------- #
+
+  # Causes `nodePackages' to use `node@20'
+  n20Overlay = final: prev: { nodejs = prev.nodejs-slim_20; };
+
+  # Add a package that requires `node@20'
+  myOverlay = final: prev: {
+    nodePackages = prev.nodePackages.extend ( nfinal: nprev: let
+      pkg-fun = { runCommand, nodejs, ... }: runCommand "bar" {
+        meta.nodeVersion = nodejs.version;
+      } ''
+        case "$( ${nodejs}/bin/node --version; )" in
+          v20.*) touch "$out"; ;;
+          v*)    echo "v20 is required" >&2; exit 1; ;;
+        esac
+      '';
+    in {
+      bar0 = final.callPackage pkg-fun {};
+      bar1 = prev.callPackage pkg-fun {};
+      bar2 = pkg-fun { inherit (prev) nodejs; inherit (final) runCommand; };
+      bar3 = pkg-fun { inherit (final) nodejs; inherit (prev) runCommand; };
+      bar4 = pkg-fun { inherit (prev) nodejs runCommand; };
+      bar5 = pkg-fun { inherit (final) nodejs runCommand; };
+    } );
+  };
+
+
+# ---------------------------------------------------------------------------- #
+
+  # Causes `nodePackages' to use `node@18'
+  n18Overlay = final: prev: { nodejs = prev.nodejs-18_x; };
+
+  # Add a package that requires `node@18'
+  friendOverlay = final: prev: {
+    nodePackages = prev.nodePackages.extend ( nfinal: nprev: let
+      pkg-fun = { runCommand, nodejs, ... }: runCommand "quux" {
+        meta.nodeVersion = nodejs.version;
+      } ''
+        case "$( ${nodejs}/bin/node --version; )" in
+          v18.*) touch "$out"; ;;
+          v*)    echo "v18 is required" >&2; exit 1; ;;
+        esac
+      '';
+    in {
+      quux0 = final.callPackage pkg-fun {};
+      quux1 = prev.callPackage pkg-fun {};
+      quux2 = pkg-fun { inherit (prev) nodejs; inherit (final) runCommand; };
+      quux3 = pkg-fun { inherit (final) nodejs; inherit (prev) runCommand; };
+      quux4 = pkg-fun { inherit (prev) nodejs runCommand; };
+      quux5 = pkg-fun { inherit (final) nodejs runCommand; };
+    } );
+  };
+
+
+# ---------------------------------------------------------------------------- #
+
+  extractNodeVersions = let
+    targets = [
+      "bar0"  "bar1"  "bar2"  "bar3"  "bar4"  "bar5"
+      "quux0" "quux1" "quux2" "quux3" "quux4" "quux5"
+    ];
+  in pkgs: let
+    get = name: {
+      inherit name;
+      value = ( builtins.getAttr name pkgs.nodePackages ).meta.nodeVersion;
+    };
+  in builtins.listToAttrs ( map get targets );
+
+
+# ---------------------------------------------------------------------------- #
+
+  showNodeVersions = pkgs: let
+    extracted = extractNodeVersions pkgs;
+    showOne   = name: version: let
+      spaces = if ( builtins.match "bar.*" name ) == null then " " else "  ";
+    in name + ":" + spaces + "nodejs@" + version;
+    strs = builtins.mapAttrs showOne extracted;
+  in builtins.concatStringsSep "\n" ( builtins.attrValues strs );
+
+
+# ---------------------------------------------------------------------------- #
+
+  chainOverlays   = builtins.foldl' ( pkgs: pkgs.extend ) pkgsFor;
+  composeOverlays = overlays:
+    pkgsFor.extend ( nixpkgs.lib.composeManyExtensions overlays );
+
+# ---------------------------------------------------------------------------- #
+
+  pkgSets = {
+    chained0  = chainOverlays   [n20Overlay myOverlay n18Overlay friendOverlay];
+    chained1  = chainOverlays   [n18Overlay friendOverlay n20Overlay myOverlay];
+    chained2  = chainOverlays   [friendOverlay n18Overlay myOverlay n20Overlay];
+    chained3  = chainOverlays   [myOverlay n20Overlay friendOverlay n18Overlay];
+    composed0 = composeOverlays [n20Overlay myOverlay n18Overlay friendOverlay];
+    composed1 = composeOverlays [n18Overlay friendOverlay n20Overlay myOverlay];
+    composed2 = composeOverlays [friendOverlay n18Overlay myOverlay n20Overlay];
+    composed3 = composeOverlays [myOverlay n20Overlay friendOverlay n18Overlay];
+  };
+
+  versions = {
+    nix = builtins.mapAttrs ( _: extractNodeVersions ) pkgSets;
+    str = builtins.mapAttrs ( _: showNodeVersions ) pkgSets;
+  };
+
+
+# ---------------------------------------------------------------------------- #
+
+in {
+
+  inherit (nixpkgs) lib;
+  inherit pkgsFor pkgSets versions;
+
+  overlays = {
+    inherit n20Overlay n18Overlay myOverlay friendOverlay;
+  };
+  util = {
+    inherit chainOverlays composeOverlays extractNodeVersions showNodeVersions;
+  };
+
+  show = str: builtins.trace ( "\n" + str ) null;
+
+  report = let
+    reportOne = name: str: ''
+      * ${name}
+        - ${builtins.replaceStrings ["\n"] ["\n  - "] str}
+    '';
+    strs = builtins.mapAttrs reportOne versions.str;
+  in builtins.concatStringsSep "\n" ( builtins.attrValues strs );
+
+}
+
+# ---------------------------------------------------------------------------- #
+#
+# Results
+# -------
+#   * chained0
+#     - bar0:  nodejs@18.16.0
+#     - bar1:  nodejs@18.16.0
+#     - bar2:  nodejs@20.2.0
+#     - bar3:  nodejs@18.16.0
+#     - bar4:  nodejs@20.2.0
+#     - bar5:  nodejs@18.16.0
+#     - quux0: nodejs@18.16.0
+#     - quux1: nodejs@18.16.0
+#     - quux2: nodejs@18.16.0
+#     - quux3: nodejs@18.16.0
+#     - quux4: nodejs@18.16.0
+#     - quux5: nodejs@18.16.0
+#
+#   * chained1
+#     - bar0:  nodejs@20.2.0
+#     - bar1:  nodejs@20.2.0
+#     - bar2:  nodejs@20.2.0
+#     - bar3:  nodejs@20.2.0
+#     - bar4:  nodejs@20.2.0
+#     - bar5:  nodejs@20.2.0
+#     - quux0: nodejs@20.2.0
+#     - quux1: nodejs@20.2.0
+#     - quux2: nodejs@18.16.0
+#     - quux3: nodejs@20.2.0
+#     - quux4: nodejs@18.16.0
+#     - quux5: nodejs@20.2.0
+#
+#   * chained2
+#     - bar0:  nodejs@20.2.0
+#     - bar1:  nodejs@20.2.0
+#     - bar2:  nodejs@18.16.0
+#     - bar3:  nodejs@20.2.0
+#     - bar4:  nodejs@18.16.0
+#     - bar5:  nodejs@20.2.0
+#     - quux0: nodejs@20.2.0
+#     - quux1: nodejs@20.2.0
+#     - quux2: nodejs@18.16.0
+#     - quux3: nodejs@20.2.0
+#     - quux4: nodejs@18.16.0
+#     - quux5: nodejs@20.2.0
+#
+#   * chained3
+#     - bar0:  nodejs@18.16.0
+#     - bar1:  nodejs@18.16.0
+#     - bar2:  nodejs@18.16.0
+#     - bar3:  nodejs@18.16.0
+#     - bar4:  nodejs@18.16.0
+#     - bar5:  nodejs@18.16.0
+#     - quux0: nodejs@18.16.0
+#     - quux1: nodejs@18.16.0
+#     - quux2: nodejs@20.2.0
+#     - quux3: nodejs@18.16.0
+#     - quux4: nodejs@20.2.0
+#     - quux5: nodejs@18.16.0
+#
+#   * composed0
+#     - bar0:  nodejs@18.16.0
+#     - bar1:  nodejs@18.16.0
+#     - bar2:  nodejs@20.2.0
+#     - bar3:  nodejs@18.16.0
+#     - bar4:  nodejs@20.2.0
+#     - bar5:  nodejs@18.16.0
+#     - quux0: nodejs@18.16.0
+#     - quux1: nodejs@18.16.0
+#     - quux2: nodejs@18.16.0
+#     - quux3: nodejs@18.16.0
+#     - quux4: nodejs@18.16.0
+#     - quux5: nodejs@18.16.0
+#
+#   * composed1
+#     - bar0:  nodejs@20.2.0
+#     - bar1:  nodejs@20.2.0
+#     - bar2:  nodejs@20.2.0
+#     - bar3:  nodejs@20.2.0
+#     - bar4:  nodejs@20.2.0
+#     - bar5:  nodejs@20.2.0
+#     - quux0: nodejs@20.2.0
+#     - quux1: nodejs@20.2.0
+#     - quux2: nodejs@18.16.0
+#     - quux3: nodejs@20.2.0
+#     - quux4: nodejs@18.16.0
+#     - quux5: nodejs@20.2.0
+#
+#   * composed2
+#     - bar0:  nodejs@20.2.0
+#     - bar1:  nodejs@20.2.0
+#     - bar2:  nodejs@18.16.0
+#     - bar3:  nodejs@20.2.0
+#     - bar4:  nodejs@18.16.0
+#     - bar5:  nodejs@20.2.0
+#     - quux0: nodejs@20.2.0
+#     - quux1: nodejs@20.2.0
+#     - quux2: nodejs@18.16.0
+#     - quux3: nodejs@20.2.0
+#     - quux4: nodejs@18.16.0
+#     - quux5: nodejs@20.2.0
+#
+#   * composed3
+#     - bar0:  nodejs@18.16.0
+#     - bar1:  nodejs@18.16.0
+#     - bar2:  nodejs@18.16.0
+#     - bar3:  nodejs@18.16.0
+#     - bar4:  nodejs@18.16.0
+#     - bar5:  nodejs@18.16.0
+#     - quux0: nodejs@18.16.0
+#     - quux1: nodejs@18.16.0
+#     - quux2: nodejs@20.2.0
+#     - quux3: nodejs@18.16.0
+#     - quux4: nodejs@20.2.0
+#     - quux5: nodejs@18.16.0
+#
+#
+# ============================================================================ #

use-cases/deep-replace.md

@@ -0,0 +1,124 @@
+# Deeply Replacing Packages in Nixpkgs
+
+## General Information
+
+Replace all "instances" of a package across a dependency graph.
+This may be replacing all usage in a package set, collection of package sets,
+a collection of ad-hoc recipes, or a collection of flakes.
+
+A possible motivation for _deep replacement_ may be to ensure that a security
+fix provided by a new release of a piece of software is used "everywhere" in
+the dependency graph.
+
+
+## Concrete Examples
+
+While the precise organization of packages will effect the complexity and
+effort required to perform _deep replacement_, in general we say that this
+is accomplished using helper functions such as `extend`, `overrideScope'`, and
+`appendOverlays`, as well as the configuration field `overlays`.
+
+### Simple Overlay
+
+```nix
+pkgs.extend ( final: prev: { foo = final.callPackage ./my-pkgs/foo {}; } )
+```
+
+
+<a name="Nested-Ex"></a>
+### Nested Overlays
+
+Nested overlays, "chained" vs. "composed" overlays, as and when to use `prev`
+and `final` are common sources of confusion.
+This can be particularly problematic when consuming overlays defined externally.
+
+There is an extended example [here](../scratch/nested-extend.nix) that shows a
+variety of common pitfalls, but the snippet below shows an abbreviated case.
+Here the use of `nodejs` is contrived, what's important is that `nodejs` is
+used to populate a nested scope `nodePackages`.
+For the purposes of this example, we'll imagine the user is unaware of how
+`nodePackages<VERSION>` attrsets should be used and naively use `nodePackages`
+to add their own package definitions.
+The core of this issue is that we attempt to merge an external overlay which
+uses `nodejs@18` with an overlay using `nodejs@20`.
+The user's goal here is really to use `nodejs@20` for the packages they define
+and for their dependencies, but doing so inadvertently modifies executables
+defined in our external overlay.
+
+External Flake:
+```nix
+{
+  outputs = { nixpkgs, ... }: let
+    # We must split these into two overlays so `prev` holds the correct `nodejs`
+    overlays.node18 = final: prev: { nodejs = prev.nodejs-18_x; };
+    overlays.theirPkgs = final: prev: {
+      nodePackages = prev.nodePackages.extend ( nfinal: nprev: {
+        # Requires `nodejs@18' to avoid runtime errors.
+        someExecutable = nfinal.callPackage ./. {};
+      } );
+    };
+    overlays.default =
+      nixpkgs.lib.composeExtensions overlays.node18 overlays.theirPkgs;
+  in { inherit overlays; }
+}
+```
+
+Our Flake:
+```
+{
+  outputs = { nixpkgs, other, ... }: let
+    overlays.deps   = other.overlays.default;
+    overlays.node20 = final: prev: { nodejs = prev.nodejs-20_x; };
+    overlays.myPkgs = final: prev: {
+      nodePackages = prev.nodePackages.extend ( nfinal: nprev: {
+        # Doesn't use any node modules from previous overlay, but requires
+        # `nodejs@20' to avoid runtime errors.
+        myModule = nfinal.callPackage ./module {};
+      } );
+      # Uses `someExecutable' from previous overlay.
+      someTool = final.callPackage ./tool {};
+    };
+    overlays.default = nixpkgs.lib.composeManyExtensions [
+      overlays.deps overlays.node20 overlays.myPkgs
+    ];
+  in {
+    inherit overlays;
+    packages.x86_64-linux = let
+      pkgsFor = nixpkgs.legacyPackages.x86_64-linux.extend overlays.default;
+    in { inherit (pkgsFor) nodePackages someTool; }
+    # packages.<SYSTEM> = ...;
+  }
+}
+```
+
+In this example the user will encounter runtime crashes in `myTool` caused by
+accidentally overriding `nodejs = nodejs-20_x;`
+in `nodePackages.someExecutable` defined externally.
+While we certainly have ways to avoid this category of issue, the process seen
+above is already a challenge to understand - ideally a more intuitive pattern
+could be offered to users.
+
+
+## Current Problems
+
+With this approach we have three main sources of complexity, none of which
+truly prevent a user from accomplishing their goal, but we might suffice to
+say that it may be worthwhile to provide a more straightforward mechanism
+for handling this use case.
+
+1. [github:NixOS/nixpkgs://lib/customisation.nix](https://github.com/NixOS/nixpkgs/blob/master/lib/customisation.nix)
+routines aren't intuitively understood by many users.
+   - `self`/`super` and `final`/`prev` are difficult for new users to understand.
+   - Easy to accidentally trigger infinite recursion.
+   - Some packages are not _truly_ overridable, which advanced users will only
+     discover after a lengthy debugging session.
+
+2. Nested scopes are difficult to locate, and the relationship between
+   parent scopes and child scopes is opaque to users.
+   - See [Nested Overlays](#Nested-Ex) example.
+
+3. With ad-hoc recipes and flakes there isn't standardized usage of
+   `overlays` that allow deep overriding of packages transitively.
+   - Improved guidance on the use of `overlays` and `follows` in `flakes`
+     could help a bit here.
+