From 4aac481f37b88db5e9184d7dde3347e74f3d3251 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 May 2023 17:36:57 +0200 Subject: [PATCH 01/10] pkgs/top-level/module: init --- pkgs/test/default.nix | 6 ++ pkgs/top-level/module/module.nix | 130 +++++++++++++++++++++++++++++++ pkgs/top-level/module/tests.nix | 59 ++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 pkgs/top-level/module/module.nix create mode 100644 pkgs/top-level/module/tests.nix diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index b6793d25b6e21..ce4afc31f517b 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -101,4 +101,10 @@ with pkgs; }; pkgs-lib = recurseIntoAttrs (import ../pkgs-lib/tests { inherit pkgs; }); + + top-level = recurseIntoAttrs { + module = recurseIntoAttrs ( + (callPackage ../top-level/module/tests.nix { }).tests + ); + }; } diff --git a/pkgs/top-level/module/module.nix b/pkgs/top-level/module/module.nix new file mode 100644 index 0000000000000..dffce13accb76 --- /dev/null +++ b/pkgs/top-level/module/module.nix @@ -0,0 +1,130 @@ +/* + A [module] for invoking Nixpkgs. + + This module is a public interface for use in third party module system + applications, so that they can re-expose the Nixpkgs configuration logic to + their users. + + It declares its options without a prefix like `nixpkgs.*`, so it is meant to + be imported in a submodule. + + This module is based on the NixOS `misc/nixpkgs.nix` module, but it gets rid + of legacy options. It may be used to replace that module after legacy options + have been deprecated and removed. + + As this module declares an option named `config`, you're recommended to pass + `shorthandOnlyDefinesConfig = true` to `types.submoduleWith`. + + module: https://nixos.org/manual/nixpkgs/unstable/#module-system +*/ +{ lib, config, options, ... }: + +let + inherit (lib) mkOption types literalExpression; + + optDefaultPrio = (lib.mkOptionDefault null).priority; + + isCross = config.buildPlatform != config.hostPlatform; + systemArgs = + if isCross + then { + localSystem = config.buildPlatform; + crossSystem = config.hostPlatform; + } + else { + localSystem = config.hostPlatform; + }; + + pkgs = + import ../default.nix ({ + inherit (config) config overlays; + } // systemArgs); + + # TODO: write a better type that checks that the definition ordering is + # unambiguous under import order. + overlayType = types.functionTo (types.functionTo types.raw); + +in +{ + options = { + hostPlatform = mkOption { + type = types.coercedTo types.str lib.systems.elaborate types.attrs; + description = lib.mdDoc '' + Specifies the platform where the package will run. + + To cross-compile, also set the `buildPlatform` option. + + Ignored when the `pkgs` option is overridden. + ''; + }; + buildPlatform = mkOption { + type = types.coercedTo types.str lib.systems.elaborate types.attrs; + description = lib.mdDoc '' + Specifies the platform where the package will be built. This is + optional; by default Nixpkgs will perform native compilation. + + Setting this option will cause NixOS to be cross-compiled. + + Ignored when the `pkgs` option is overridden. + ''; + defaultText = literalExpression ''hostPlatform''; + default = config.hostPlatform; + }; + overlays = mkOption { + default = [ ]; + example = literalExpression + '' + [ + (self: super: { + openssh = super.openssh.override { + hpnSupport = true; + kerberos = self.libkrb5; + }; + }) + ] + ''; + type = types.listOf overlayType; + description = lib.mdDoc '' + List of overlays to use with Nixpkgs. + (For details, see the Nixpkgs documentation.) It allows + you to override packages globally. Each function in the list + takes as an argument the *original* Nixpkgs. + The first argument should be used for finding dependencies, and + the second should be used for overriding recipes. + + Ignored when `pkgs` option is overridden. + ''; + }; + config = mkOption { + default = { }; + example = literalExpression + '' + { allowBroken = true; allowUnfree = true; } + ''; + # FIXME: use a mergeable type, like NixOS but not a hack + type = types.unique { message = "nixpkgs/pkgs/top-level/module.nix: Merging is not supported for the Nixpkgs config option yet."; } types.attrs; + description = lib.mdDoc '' + The configuration of the Nix Packages collection. (For + details, see the Nixpkgs documentation.) It allows you to set + package configuration options. + + Ignored when the `pkgs` option is overridden. + ''; + }; + pkgs = mkOption { + type = types.pkgs; + description = lib.mdDoc '' + This option's value is initialized by its siblings `hostPlatform`, `overlays`, `config`, etc. + + It contains the Nixpkgs package set. [The package search on search.nixos.org](https://search.nixos.org/packages) lists virtually all packages in this set, although `overlays` can change and add packages. + + You could override the value if you have specific instantiation of Nixpkgs + that you would like to use, but note that this will cause those sibling options + to be ignored. + ''; + }; + }; + config = { + inherit pkgs; + }; +} diff --git a/pkgs/top-level/module/tests.nix b/pkgs/top-level/module/tests.nix new file mode 100644 index 0000000000000..9cf4c00c369d0 --- /dev/null +++ b/pkgs/top-level/module/tests.nix @@ -0,0 +1,59 @@ +/* + Run + [nixpkgs]$ nix-build -A tests.top-level.module --show-trace + + Repl + [nixpkgs]$ nix repl pkgs/top-level/module/tests.nix --show-trace + nix-repl> tests + "ok" +*/ + +# This function is `callPackage`d, but we'd like to load it in nix repl as well, +# so we add some defaults. +{ lib ? import ../../../lib, emptyFile ? "ok" }: + +# We define use a rec {} mostly as a let binding, as the tests attribute is all +# that really matters for running the tests. However, having access to all the +# other attributes is useful for debugging. +rec { + inherit (lib) evalModules; + + invokeModule = callerModule: evalModules { modules = [ ./module.nix callerModule ]; }; + + basic = invokeModule { hostPlatform = "x86_64-linux"; }; + basicExpect = import ../default.nix { localSystem = "x86_64-linux"; }; + + cross = invokeModule { buildPlatform = "x86_64-linux"; hostPlatform = "aarch64-linux"; }; + crossExpect = import ../default.nix { localSystem = "x86_64-linux"; crossSystem = "aarch64-linux"; }; + + overlayExample = self: super: { hello = super.hello.overrideAttrs (old: { name = "helloFromOverlay"; }); }; + + overlay = invokeModule { + hostPlatform = "x86_64-linux"; + overlays = [ overlayExample ]; + }; + # overlayExpect = import ../default.nix { localSystem = "x86_64-linux"; overlays = [ overlayExample ]; }; + + havingConfig = invokeModule { + # Defining a value for an option named `config` is a bit awkward withou + # the shorthand syntax. This is noted in the module file's top comment. + config = { + hostPlatform = "x86_64-linux"; + config = { + # Not the most popular config option, but one that is easy to test. + perlPackageOverrides = pkgs: { just-a-proof-of-use = true; }; + }; + }; + }; + + tests = + assert basic.config.pkgs.hello == basicExpect.hello; + + assert cross.config.pkgs.hello == crossExpect.hello; + + assert overlay.config.pkgs.hello.name == "helloFromOverlay"; + + assert havingConfig.config.pkgs.perlPackages.just-a-proof-of-use; + + emptyFile; +} From bfe8fd23c8db4de5237c5289a0688329dfbab985 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 May 2023 19:33:03 +0200 Subject: [PATCH 02/10] flake.nix: Add modules.generic.nixpkgs A helper for module system applications. Naming compatible with https://github.com/NixOS/nix/issues/6257 To be supported by nix flake check in https://github.com/NixOS/nix/pull/8332 Concept is in the spirit of a proposed module based solution to [RFC 0078 System-agnostic configuration file generators](https://github.com/NixOS/rfcs/pull/78). --- doc/using/configuration.chapter.md | 7 +++++++ flake.nix | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/doc/using/configuration.chapter.md b/doc/using/configuration.chapter.md index 8d246b117b05f..69f5f8f191cc6 100644 --- a/doc/using/configuration.chapter.md +++ b/doc/using/configuration.chapter.md @@ -191,6 +191,13 @@ list-id: configuration-variable-list source: ../config-options.json ``` +## Calling Nixpkgs from the Module System {#sec-invoke-nixpkgs-by-module} + +For [module system](#module-system) application authors, Nixpkgs provides a reusable module for the purpose of invoking Nixpkgs. Most users should not have to be aware of this. The documentation for its options is meant to be re-exposed in the application's documentation and is therefore not repeated here. + +Applications can import the implementation file [`"${nixpkgs}/pkgs/top-level/module/module.nix"`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/module/module.nix). + +The flake attribute (experimental) for this module is `.modules.generic.nixpkgs`. ## Declarative Package Management {#sec-declarative-package-management} diff --git a/flake.nix b/flake.nix index fa00bffcdf92f..715c0c0b8dd7d 100644 --- a/flake.nix +++ b/flake.nix @@ -71,5 +71,12 @@ */ readOnlyPkgs = ./nixos/modules/misc/nixpkgs/read-only.nix; }; + + modules = { + /* Modules that are not specific to any application of the module system. */ + generic = { + nixpkgs = ./pkgs/top-level/module/module.nix; + }; + }; }; } From 740953402c9a89297885bc74be70702d2ea68b68 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 May 2023 20:50:39 +0200 Subject: [PATCH 03/10] modules.generic.nixpkgs: Add optimization machinery --- pkgs/top-level/module/module.nix | 52 ++++++++++++++++++++++++--- pkgs/top-level/module/tests.nix | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/pkgs/top-level/module/module.nix b/pkgs/top-level/module/module.nix index dffce13accb76..c8382492b4cdf 100644 --- a/pkgs/top-level/module/module.nix +++ b/pkgs/top-level/module/module.nix @@ -36,14 +36,34 @@ let }; pkgs = - import ../default.nix ({ - inherit (config) config overlays; - } // systemArgs); + import ../default.nix ( + # NOTE: make inputsSet covers all parameters; see _memoize + # Convenience options that write to inputsSet should be omitted. + # This point is that inputsSet must form a unique key so that + # memoization optimizations are safe. + { + inherit (config) config overlays; + } // systemArgs + ); # TODO: write a better type that checks that the definition ordering is # unambiguous under import order. overlayType = types.functionTo (types.functionTo types.raw); + # A unique key for the Nixpkgs package set to be instantiated. + # + # This is only relevant for _memoize - ie when you're writing or troubleshooting + # a module system application that uses _memoize to memoize things. + # + # Default values and unset values are omitted so that future additions to the + # key are usable with _memoize implementations that are not aware of them, + # as long as these additions are unused. + inputsSet = lib.filterAttrs + (_: v: v != v.isDefined && v.highestPrio < optDefaultPrio) + { + inherit (options) hostPlatform buildPlatform overlays config; + }; + in { options = { @@ -123,8 +143,32 @@ in to be ignored. ''; }; + _memoize = mkOption { + type = types.functionTo types.pkgs; + description = lib.mdDoc '' + Memoization is a caching technique employed in functional languages, by putting thunks or results in a shared data structure. + + Reusing the same Nixpkgs instance in multiple places can save a bit of time and a lot of memory. + + `_memoize` is a hook that allows you to "alter" the Nixpkgs package that is generated for the `pkgs` value. + This *should* only be done in accordance with other options such as `hostPlatform`, `overlays`, and `config`, but it is not enforced. It is *your* responsibility. To help you out a bit, this hook is passed `inputsSet`, which contains the options that this module knows your function must take into account. If your implementation does not recognize one of these options, it must return the `pkgs` argument. + + The Nixpkgs flake uses this to return the actual `legacyPackages.''${hostPlatform}` set *if* an equivalent Nixpkgs set would be constructed - ie the other options are all defaults. + ''; + internal = true; + default = { pkgs, ... }: pkgs; + example = literalExpression '' + { pkgs, inputsSet, ... }: + if lib.mapAttrs (_: _: null) (builtins.removeAttrs inputsSet ["hostPlatform"]) + != { hostPlatform = null; } + then pkgs + else + # Get a shared instance + legacyPackages.''${hostPlatform} + ''; + }; }; config = { - inherit pkgs; + pkgs = config._memoize { inherit pkgs inputsSet; }; }; } diff --git a/pkgs/top-level/module/tests.nix b/pkgs/top-level/module/tests.nix index 9cf4c00c369d0..653eb800ddacf 100644 --- a/pkgs/top-level/module/tests.nix +++ b/pkgs/top-level/module/tests.nix @@ -18,6 +18,15 @@ rec { inherit (lib) evalModules; + # Helper functions + noValues = lib.mapAttrs (_: _: null); + shouldBe = actual: expected: if actual == expected then true else + builtins.trace "actual:" + builtins.trace actual + builtins.trace "expected:" + builtins.trace expected + false; + invokeModule = callerModule: evalModules { modules = [ ./module.nix callerModule ]; }; basic = invokeModule { hostPlatform = "x86_64-linux"; }; @@ -46,6 +55,36 @@ rec { }; }; + optimizedSimple = invokeModule { + hostPlatform = "x86_64-linux"; + _memoize = args@{ pkgs, inputsSet, ... }: + # Don't do this in production. _memoize should be transparent. + pkgs // { testData = args; }; + }; + + optimizedAllOpts = invokeModule { + config = { + buildPlatform = "x86_64-linux"; + hostPlatform = "aarch64-linux"; + config = { allowAliases = false; allowUnfree = true; }; + overlays = [ overlayExample ]; + _memoize = args@{ pkgs, inputsSet, ... }: + # Don't do this in production. _memoize should be transparent. + pkgs // { testData = args; }; + }; + }; + + # All inputs that affect the construction of the package set / arguments to _memoize. + # Not `_memoize` because it is not an argument to itself. + # Not `pkgs` because it is not an argument and the result of the `_memoize` call. + # Not any options that are implemented by only setting one or more of the already listed options (and would be possible to move to a separate module). See the test case with removeAttrs below. + allInputsSet = { + buildPlatform = null; + hostPlatform = null; + config = null; + overlays = null; + }; + tests = assert basic.config.pkgs.hello == basicExpect.hello; @@ -55,5 +94,27 @@ rec { assert havingConfig.config.pkgs.perlPackages.just-a-proof-of-use; + assert noValues optimizedSimple.config.pkgs.testData.inputsSet == { hostPlatform = null; }; + + assert noValues optimizedAllOpts.config.pkgs.testData.inputsSet == allInputsSet; + + # Make sure that all necessary options are part of inputsSet. This will fail + # if an option is added without considering this. If the new option is a new + # addition to the unique key, it should be added to allInputsSet. + assert shouldBe + (builtins.removeAttrs (noValues optimizedAllOpts.options) [ + # All options that do not directly affect the construction of pkgs. + # If a new option's effect is implemented by setting one of the options + # already in `inputsSet`, it does not need to be listed here. + # + # See comment on `allInputsSet`. + # Defined by us + "pkgs" "_memoize" + + # From the module system itself + "_module" + ]) + allInputsSet; + emptyFile; } From 1fad67687540b3965abdd006e48b67b9c231c515 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 May 2023 22:30:26 +0200 Subject: [PATCH 04/10] flake.nix: Optimize modules.generic.nixpkgs when pkgs is legacyPackages --- flake.nix | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 715c0c0b8dd7d..f08f8ed3b6eef 100644 --- a/flake.nix +++ b/flake.nix @@ -75,7 +75,25 @@ modules = { /* Modules that are not specific to any application of the module system. */ generic = { - nixpkgs = ./pkgs/top-level/module/module.nix; + nixpkgs = { + imports = [ + ./pkgs/top-level/module/module.nix + ]; + config = { + _memoize = { pkgs, inputsSet, ... }: + let system = lib.systems.toLosslessStringMaybe inputsSet.hostPlatform.value.system; + in + if lib.attrNames inputsSet == [ "hostPlatform" ] && system != null + then + # As only hostPlatform matters in this invocation, we can save + # memory by sharing the Nixpkgs invocation. + # `pkgs` is never evaluated. + self.legacyPackages.${system} + else + # We let the module invoke Nixpkgs as usual. + pkgs; + }; + }; }; }; }; From 14400cdfcead8c508abc76be2c48b77d6c4bf2ee Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 May 2023 18:08:29 +0200 Subject: [PATCH 05/10] nixos/qemu-vm.nix: Use types.pkgs directly This hack isn't needed anymore, and would be incompatible with having submodule at `nixpkgs.*`. --- nixos/modules/virtualisation/qemu-vm.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 4aac0fa90e8bd..afcaf8cd05100 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -630,7 +630,7 @@ in }; virtualisation.host.pkgs = mkOption { - type = options.nixpkgs.pkgs.type; + type = types.pkgs; default = pkgs; defaultText = literalExpression "pkgs"; example = literalExpression '' From 20e482a2408d21a0ccc61c06f786342a23f8bb15 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 May 2023 00:03:42 +0200 Subject: [PATCH 06/10] nixos/doc: Add modules.generic.nixpkgs to release notes --- nixos/doc/manual/release-notes/rl-2311.section.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 94471eddf1688..7c38a7dd2ea6a 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -82,6 +82,9 @@ - A new option was added to the virtualisation module that enables specifying explicitly named network interfaces in QEMU VMs. The existing `virtualisation.vlans` is still supported for cases where the name of the network interface is irrelevant. +- The flake now exposes a module [`modules.generic.nixpkgs`](https://nixos.org/manual/nixpkgs/unstable/#sec-invoke-nixpkgs-by-module) for configuring and invoking Nixpkgs, to be used in new or existing module system applications. + It is more efficient than a simple reimplementation, and it carries none of the NixOS legacy options. + - DocBook option documentation is no longer supported, all module documentation now uses markdown. - `services.fail2ban.jails` can now be configured with attribute sets defining settings and filters instead of lines. The stringed options `daemonConfig` and `extraSettings` have respectively been replaced by `daemonSettings` and `jails.DEFAULT.settings` which use attribute sets. From 44930c1d8f19c1a1c70480dfb4352f9ae942ba13 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 May 2023 18:08:00 +0200 Subject: [PATCH 07/10] nixosModules.noLegacyPkgs: init --- flake.nix | 41 ++++++++++++------- .../manual/release-notes/rl-2311.section.md | 14 +++++++ nixos/modules/misc/nixpkgs/no-legacy.nix | 39 ++++++++++++++++++ nixos/modules/misc/nixpkgs/read-only.nix | 1 + nixos/modules/misc/nixpkgs/test.nix | 11 +++++ 5 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 nixos/modules/misc/nixpkgs/no-legacy.nix diff --git a/flake.nix b/flake.nix index f08f8ed3b6eef..961293b79feee 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,20 @@ lib = import ./lib; forAllSystems = lib.genAttrs lib.systems.flakeExposed; + + pkgsMemoizer = { pkgs, inputsSet, ... }: + let system = lib.systems.toLosslessStringMaybe inputsSet.hostPlatform.value.system; + in + if lib.attrNames inputsSet == [ "hostPlatform" ] && system != null + then + # As only hostPlatform matters in this invocation, we can save + # memory by sharing the Nixpkgs invocation. + # `pkgs` is never evaluated. + self.legacyPackages.${system} + else + # We let the module invoke Nixpkgs as usual. + pkgs; + in { lib = lib.extend (final: prev: { @@ -70,6 +84,18 @@ } */ readOnlyPkgs = ./nixos/modules/misc/nixpkgs/read-only.nix; + + /* + Replaces the module for `nixpkgs.*` by a simpler module that can + reuse `legacyPackages`, but does not have any backwards compatibility, + so no `nixpkgs.*System`, etc. + + Ignored when `readOnlyPkgs` is imported. + */ + noLegacyPkgs = { + imports = [ ./nixos/modules/misc/nixpkgs/no-legacy.nix ]; + nixpkgs._memoize = pkgsMemoizer; + }; }; modules = { @@ -79,20 +105,7 @@ imports = [ ./pkgs/top-level/module/module.nix ]; - config = { - _memoize = { pkgs, inputsSet, ... }: - let system = lib.systems.toLosslessStringMaybe inputsSet.hostPlatform.value.system; - in - if lib.attrNames inputsSet == [ "hostPlatform" ] && system != null - then - # As only hostPlatform matters in this invocation, we can save - # memory by sharing the Nixpkgs invocation. - # `pkgs` is never evaluated. - self.legacyPackages.${system} - else - # We let the module invoke Nixpkgs as usual. - pkgs; - }; + config._memoize = pkgsMemoizer; }; }; }; diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 7c38a7dd2ea6a..5e213971c63d6 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -85,6 +85,20 @@ - The flake now exposes a module [`modules.generic.nixpkgs`](https://nixos.org/manual/nixpkgs/unstable/#sec-invoke-nixpkgs-by-module) for configuring and invoking Nixpkgs, to be used in new or existing module system applications. It is more efficient than a simple reimplementation, and it carries none of the NixOS legacy options. +- Importable modules have been added for flakes users to optimize `pkgs` in NixOS. + + - To reuse the `legacyPackages.*` instantiation of Nixpkgs *if applicable*, use. + ```nix + imports = [ nixpkgs.nixosModules.noLegacyPkgs ]; + ``` + Here, legacy refers to old `nixpkgs.*` options. + + - If you want to be sure that a shared `pkgs` instance is reused, you may invoke Nixpkgs yourself and use `readOnlyPkgs`, which makes sure the other `nixpkgs.*` options are unset. + ```nix + imports = [ nixpkgs.nixosModules.readOnlyPkgs ]; + nixpkgs.pkgs = ...; + ``` + - DocBook option documentation is no longer supported, all module documentation now uses markdown. - `services.fail2ban.jails` can now be configured with attribute sets defining settings and filters instead of lines. The stringed options `daemonConfig` and `extraSettings` have respectively been replaced by `daemonSettings` and `jails.DEFAULT.settings` which use attribute sets. diff --git a/nixos/modules/misc/nixpkgs/no-legacy.nix b/nixos/modules/misc/nixpkgs/no-legacy.nix new file mode 100644 index 0000000000000..3558e7f07dc6d --- /dev/null +++ b/nixos/modules/misc/nixpkgs/no-legacy.nix @@ -0,0 +1,39 @@ +/* + A clean, [RFC 0078] style implementation of the NixOS `nixpkgs.*` module, which + is free of legacy options (as of 23.05). + + [RFC 0078]: https://github.com/NixOS/rfcs/pull/78 +*/ + +{ config, lib, ... }: + +{ + disabledModules = [ + # This replaces the traditional nixpkgs module + ../nixpkgs.nix + ]; + options = { + nixpkgs = lib.mkOption { + description = lib.mdDoc '' + Options that configure the `pkgs` module argument. + ''; + example = lib.literalExpression '' + nixpkgs = { + config.allowUnfree = true; + overlays = [ + (final: prev: { + # Patch nixpkgs here + }) + ]; + }; + ''; + type = lib.types.submoduleWith { + shorthandOnlyDefinesConfig = true; + modules = [ ../../../../pkgs/top-level/module/module.nix ]; + }; + }; + }; + config = { + _module.args.pkgs = config.nixpkgs.pkgs; + }; +} diff --git a/nixos/modules/misc/nixpkgs/read-only.nix b/nixos/modules/misc/nixpkgs/read-only.nix index 2a783216a9d54..217db69a39322 100644 --- a/nixos/modules/misc/nixpkgs/read-only.nix +++ b/nixos/modules/misc/nixpkgs/read-only.nix @@ -19,6 +19,7 @@ in { disabledModules = [ ../nixpkgs.nix + ./no-legacy.nix ]; options = { nixpkgs = { diff --git a/nixos/modules/misc/nixpkgs/test.nix b/nixos/modules/misc/nixpkgs/test.nix index 0536cfc9624a2..7fb686f442375 100644 --- a/nixos/modules/misc/nixpkgs/test.nix +++ b/nixos/modules/misc/nixpkgs/test.nix @@ -68,6 +68,11 @@ let nixpkgs.buildPlatform = "foo-linux"; # do in pkgs instead! }; + noLegacy = evalMinimalConfig { + imports = [ ./no-legacy.nix ]; + nixpkgs.hostPlatform = "aarch64-linux"; + }; + throws = x: ! (builtins.tryEval x).success; in @@ -124,5 +129,11 @@ lib.recurseIntoAttrs { assert !readOnly.options.nixpkgs?localSystem; assert !readOnly.options.nixpkgs?crossSystem; + assert noLegacy._module.args.pkgs.stdenv.hostPlatform.system == "aarch64-linux"; + assert noLegacy._module.args.pkgs.stdenv.buildPlatform.system == "aarch64-linux"; + assert lib.systems.equals noLegacy._module.args.pkgs.stdenv.hostPlatform withHost._module.args.pkgs.stdenv.hostPlatform; + assert lib.systems.equals noLegacy._module.args.pkgs.stdenv.buildPlatform withHost._module.args.pkgs.stdenv.buildPlatform; + assert noLegacy._module.args.pkgs.hello == withHost._module.args.pkgs.hello; + pkgs.emptyFile; } From fa1a58df165b0430281a9f714c220afee62045e9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 May 2023 18:59:26 +0200 Subject: [PATCH 08/10] nixos/doc: Add Optional Modules --- flake.nix | 22 ++------- nixos/doc/manual/default.nix | 54 ++++++++++++++++++---- nixos/doc/manual/manual.md | 4 ++ nixos/doc/manual/nixos-optional-modules.md | 53 +++++++++++++++++++++ 4 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 nixos/doc/manual/nixos-optional-modules.md diff --git a/flake.nix b/flake.nix index 961293b79feee..024ee450a8026 100644 --- a/flake.nix +++ b/flake.nix @@ -72,26 +72,12 @@ nixosModules = { notDetected = ./nixos/modules/installer/scan/not-detected.nix; - /* - Make the `nixpkgs.*` configuration read-only. Guarantees that `pkgs` - is the way you initialize it. - - Example: - - { - imports = [ nixpkgs.nixosModules.readOnlyPkgs ]; - nixpkgs.pkgs = nixpkgs.legacyPackages.x86_64-linux; - } - */ + # See nixos/doc/manual/nixos-optional-modules.md + # TODO: online manual link readOnlyPkgs = ./nixos/modules/misc/nixpkgs/read-only.nix; - /* - Replaces the module for `nixpkgs.*` by a simpler module that can - reuse `legacyPackages`, but does not have any backwards compatibility, - so no `nixpkgs.*System`, etc. - - Ignored when `readOnlyPkgs` is imported. - */ + # See nixos/doc/manual/nixos-optional-modules.md + # TODO: online manual link noLegacyPkgs = { imports = [ ./nixos/modules/misc/nixpkgs/no-legacy.nix ]; nixpkgs._memoize = pkgsMemoizer; diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 40af4c1fa0b06..3cb8029031a51 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -37,16 +37,7 @@ let nixos-lib = import ../../lib { }; - testOptionsDoc = let - eval = nixos-lib.evalTest { - # Avoid evaluating a NixOS config prototype. - config.node.type = lib.types.deferredModule; - options._module.args = lib.mkOption { internal = true; }; - }; - in buildPackages.nixosOptionsDoc { - inherit (eval) options; - inherit revision; - transformOptions = opt: opt // { + cleanupLocations = opt: opt // { # Clean up declaration sites to not refer to the NixOS source tree. declarations = map @@ -58,11 +49,46 @@ let else decl) opt.declarations; }; + + testOptionsDoc = let + eval = nixos-lib.evalTest { + # Avoid evaluating a NixOS config prototype. + config.node.type = lib.types.deferredModule; + options._module.args = lib.mkOption { internal = true; }; + }; + in buildPackages.nixosOptionsDoc { + inherit (eval) options; + inherit revision; + transformOptions = cleanupLocations; documentType = "none"; variablelistId = "test-options-list"; optionIdPrefix = "test-opt-"; }; + optionalDocs = lib.mapAttrs (name: module: + let + # This is quite simple for now, but may need stubs for more complex modules. + eval = nixos-lib.evalModules { + modules = [ + module + { options._module.args = lib.mkOption { internal = true; }; } + ]; + }; + in + buildPackages.nixosOptionsDoc { + inherit (eval) options; + inherit revision; + transformOptions = cleanupLocations; + # These are for direct to docbook generation, which we don't use here. + documentType = throw "documentType not set"; + variablelistId = throw "variablelistId not set"; + optionIdPrefix = throw "optionIdPrefix not set"; + } + ) { + readOnlyPkgs = ../../modules/misc/nixpkgs/read-only.nix; + noLegacyPkgs = ../../modules/misc/nixpkgs/no-legacy.nix; + }; + prepareManualFromMD = '' cp -r --no-preserve=all $inputs/* . @@ -76,6 +102,14 @@ let --replace \ '@NIXOS_OPTIONS_JSON@' \ ${optionsDoc.optionsJSON}/share/doc/nixos/options.json + substituteInPlace ./nixos-optional-modules.md \ + --replace \ + '@OPTIONS_JSON_noLegacyPkgs@' \ + ${optionalDocs.noLegacyPkgs.optionsJSON}/share/doc/nixos/options.json \ + --replace \ + '@OPTIONS_JSON_readOnlyPkgs@' \ + ${optionalDocs.readOnlyPkgs.optionsJSON}/share/doc/nixos/options.json \ + ; substituteInPlace ./development/writing-nixos-tests.section.md \ --replace \ '@NIXOS_TEST_OPTIONS_JSON@' \ diff --git a/nixos/doc/manual/manual.md b/nixos/doc/manual/manual.md index 8cb766eeccf64..cd9c23ffdd58c 100644 --- a/nixos/doc/manual/manual.md +++ b/nixos/doc/manual/manual.md @@ -51,6 +51,10 @@ contributing-to-this-manual.chapter.md nixos-options.md ``` +```{=include=} appendix html:into-file=//optional-modules.html +nixos-optional-modules.md +``` + ```{=include=} appendix html:into-file=//release-notes.html release-notes/release-notes.md ``` diff --git a/nixos/doc/manual/nixos-optional-modules.md b/nixos/doc/manual/nixos-optional-modules.md new file mode 100644 index 0000000000000..c00a0033c5693 --- /dev/null +++ b/nixos/doc/manual/nixos-optional-modules.md @@ -0,0 +1,53 @@ +# Optional Modules {#ch-optional-modules} + +## `noLegacyPkgs` {#sec-noLegacyPkgs} + +This module reimplements `nixpkgs.*` without legacy options such as `nixpkgs.localSystem`. + +When imported from the flake (experimental), this module will reuse `legacyPackages` when only `hostPlatform` is specified. + +This module is ignored when [`readOnlyPkgs`](#sec-readOnlyPkgs) is imported. + +Example use with a flake (experimental): + +```nix +inputs.nixpkgs.lib.nixosSystem { + modules = [ + inputs.nixpkgs.nixosModules.noLegacyPkgs + ./configuration.nix + ]; +} +``` + +```{=include=} options +id-prefix: module-noLegacyPkgs-opt- +list-id: module-noLegacyPkgs-options +source: @OPTIONS_JSON_noLegacyPkgs@ +``` + +## `readOnlyPkgs` {#sec-readOnlyPkgs} + +This module ensures that the `pkgs` module argument is as specified, manually. + +The usual `nixpkgs.*` options are replaced by read-only options for compatibility. These are not listed here. NixOS modules should +get their information from the `pkgs` module argument and ignore `nixpkgs.*`. + +Example use with a flake (experimental): + +```nix +inputs.nixpkgs.lib.nixosSystem { + modules = [ + inputs.nixpkgs.nixosModules.readOnlyPkgs + ./configuration.nix + { + nixpkgs.pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux; + } + ]; +} +``` + +```{=include=} options +id-prefix: module-readOnlyPkgs-opt- +list-id: module-readOnlyPkgs-options +source: @OPTIONS_JSON_readOnlyPkgs@ +``` From 01df1f82d2d255a48e07ea290a197a2e0f579ec5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 May 2023 23:35:25 +0200 Subject: [PATCH 09/10] nixos/doc/manual: Set feature flag and explain minimalModules use A feature flag warning was rightfully picked up by ofborg. This should solve it, and hopefully help out the next person who tries to render docs for an optional module. --- nixos/doc/manual/default.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 3cb8029031a51..c1e917f3787f4 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -35,7 +35,12 @@ let }; }; - nixos-lib = import ../../lib { }; + nixos-lib = import ../../lib { featureFlags = { + # We use a minimal module list to evaluate the docs of the extra modules. + # This is significantly faster than loading all the modules, and we can pull + # this off because we don't require any dependencies to be loaded. + minimalModules = { }; }; + }; cleanupLocations = opt: opt // { # Clean up declaration sites to not refer to the NixOS source tree. @@ -85,6 +90,10 @@ let optionIdPrefix = throw "optionIdPrefix not set"; } ) { + # NOTE: These don't have to be paths. If a module needs dependencies to be loaded + # for doc rendering, do something like + # newModule = { imports = [ ../../modules/new.nix ../../modules/dep.nix ]; } + # or import it transitively. readOnlyPkgs = ../../modules/misc/nixpkgs/read-only.nix; noLegacyPkgs = ../../modules/misc/nixpkgs/no-legacy.nix; }; From 4f51d81b0aa49bb87e55c3cc12e3e7bd7670ebb0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jul 2023 23:56:44 +0200 Subject: [PATCH 10/10] doc/using/configuration: Update Co-authored-by: David Arnold --- doc/using/configuration.chapter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/using/configuration.chapter.md b/doc/using/configuration.chapter.md index 69f5f8f191cc6..38ef21a40b1eb 100644 --- a/doc/using/configuration.chapter.md +++ b/doc/using/configuration.chapter.md @@ -193,7 +193,7 @@ source: ../config-options.json ## Calling Nixpkgs from the Module System {#sec-invoke-nixpkgs-by-module} -For [module system](#module-system) application authors, Nixpkgs provides a reusable module for the purpose of invoking Nixpkgs. Most users should not have to be aware of this. The documentation for its options is meant to be re-exposed in the application's documentation and is therefore not repeated here. +For [module system](#module-system) application authors, Nixpkgs provides a reusable module for the purpose of configuring Nixpkgs. Other users should not have to be aware of this. Therefore, the documentation of its options is not rendered here. Module system authors should re-expose it in their own documentation. Applications can import the implementation file [`"${nixpkgs}/pkgs/top-level/module/module.nix"`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/module/module.nix).