From dece37b83a46d488787859332e18286727b96cc4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 27 Oct 2021 20:42:05 +0200 Subject: [PATCH 1/7] lib.evalModules: Add extendModules and type to result Allows the simultaneous construction of top-level invocations and submodule types. This helps structure configuration systems integration code. --- lib/modules.nix | 27 +++++++-- lib/tests/modules.sh | 7 +++ .../declare-submodule-via-evalModules.nix | 28 ++++++++++ lib/types.nix | 56 +++++++++---------- 4 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 lib/tests/modules/declare-submodule-via-evalModules.nix diff --git a/lib/modules.nix b/lib/modules.nix index 46ae3f136310b..c25972999dfe2 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -60,7 +60,8 @@ rec { it is to transparently move a set of modules to be a submodule of another config (as the proper arguments need to be replicated at each call to evalModules) and the less declarative the module set is. */ - evalModules = { modules + evalModules = evalModulesArgs@ + { modules , prefix ? [] , # This should only be used for special arguments that need to be evaluated # when resolving module structure (like in imports). For everything else, @@ -183,10 +184,26 @@ rec { else throw baseMsg else null; - result = builtins.seq checkUnmatched { - inherit options; - config = removeAttrs config [ "_module" ]; - inherit (config) _module; + checked = builtins.seq checkUnmatched; + + result = { + options = checked options; + config = checked (removeAttrs config [ "_module" ]); + _module = checked (config._module); + + extendModules = extendArgs@{ + modules ? [], + specialArgs ? {}, + prefix ? [], + }: + evalModules (evalModulesArgs // { + modules = evalModulesArgs.modules ++ modules; + specialArgs = evalModulesArgs.specialArgs or {} // specialArgs; + prefix = extendArgs.prefix or evalModulesArgs.prefix; + }); + type = lib.types.submoduleWith { + inherit modules specialArgs; + }; }; in result; diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index b51db91f6b078..49fc8bcbafc43 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -179,6 +179,13 @@ checkConfigOutput "true" config.submodule.outer ./declare-submoduleWith-modules. # which evaluates all the modules defined by the type) checkConfigOutput "submodule" options.submodule.type.description ./declare-submoduleWith-modules.nix +## submodules can be declared using (evalModules {...}).type +checkConfigOutput "true" config.submodule.inner ./declare-submodule-via-evalModules.nix +checkConfigOutput "true" config.submodule.outer ./declare-submodule-via-evalModules.nix +# Should also be able to evaluate the type name (which evaluates freeformType, +# which evaluates all the modules defined by the type) +checkConfigOutput "submodule" options.submodule.type.description ./declare-submodule-via-evalModules.nix + ## Paths should be allowed as values and work as expected checkConfigOutput "true" config.submodule.enable ./declare-submoduleWith-path.nix diff --git a/lib/tests/modules/declare-submodule-via-evalModules.nix b/lib/tests/modules/declare-submodule-via-evalModules.nix new file mode 100644 index 0000000000000..2841c64a073d5 --- /dev/null +++ b/lib/tests/modules/declare-submodule-via-evalModules.nix @@ -0,0 +1,28 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + inherit (lib.evalModules { + modules = [ + { + options.inner = lib.mkOption { + type = lib.types.bool; + default = false; + }; + } + ]; + }) type; + default = {}; + }; + + config.submodule = lib.mkMerge [ + ({ lib, ... }: { + options.outer = lib.mkOption { + type = lib.types.bool; + default = false; + }; + }) + { + inner = true; + outer = true; + } + ]; +} diff --git a/lib/types.nix b/lib/types.nix index c2532065d7eac..244cbb6b5354d 100644 --- a/lib/types.nix +++ b/lib/types.nix @@ -505,17 +505,36 @@ rec { then setFunctionArgs (args: unify (value args)) (functionArgs value) else unify (if shorthandOnlyDefinesConfig then { config = value; } else value); - allModules = defs: modules ++ imap1 (n: { value, file }: + allModules = defs: imap1 (n: { value, file }: if isAttrs value || isFunction value then # Annotate the value with the location of its definition for better error messages coerce (lib.modules.unifyModuleSyntax file "${toString file}-${toString n}") value else value ) defs; - freeformType = (evalModules { - inherit modules specialArgs; - args.name = "‹name›"; - })._module.freeformType; + base = evalModules { + inherit specialArgs; + modules = [{ + # This is a work-around for the fact that some sub-modules, + # such as the one included in an attribute set, expects an "args" + # attribute to be given to the sub-module. As the option + # evaluation does not have any specific attribute name yet, we + # provide a default for the documentation and the freeform type. + # + # This is necessary as some option declaration might use the + # "name" attribute given as argument of the submodule and use it + # as the default of option declarations. + # + # We use lookalike unicode single angle quotation marks because + # of the docbook transformation the options receive. In all uses + # > and < wouldn't be encoded correctly so the encoded values + # would be used, and use of `<` and `>` would break the XML document. + # It shouldn't cause an issue since this is cosmetic for the manual. + _module.args.name = lib.mkOptionDefault "‹name›"; + }] ++ modules; + }; + + freeformType = base._module.freeformType; in mkOptionType rec { @@ -523,32 +542,13 @@ rec { description = freeformType.description or name; check = x: isAttrs x || isFunction x || path.check x; merge = loc: defs: - (evalModules { - modules = allModules defs; - inherit specialArgs; - args.name = last loc; + (base.extendModules { + modules = [ { _module.args.name = last loc; } ] ++ allModules defs; prefix = loc; }).config; emptyValue = { value = {}; }; - getSubOptions = prefix: (evalModules - { inherit modules prefix specialArgs; - # This is a work-around due to the fact that some sub-modules, - # such as the one included in an attribute set, expects a "args" - # attribute to be given to the sub-module. As the option - # evaluation does not have any specific attribute name, we - # provide a default one for the documentation. - # - # This is mandatory as some option declaration might use the - # "name" attribute given as argument of the submodule and use it - # as the default of option declarations. - # - # Using lookalike unicode single angle quotation marks because - # of the docbook transformation the options receive. In all uses - # > and < wouldn't be encoded correctly so the encoded values - # would be used, and use of `<` and `>` would break the XML document. - # It shouldn't cause an issue since this is cosmetic for the manual. - args.name = "‹name›"; - }).options // optionalAttrs (freeformType != null) { + getSubOptions = prefix: (base.extendModules + { inherit prefix; }).options // optionalAttrs (freeformType != null) { # Expose the sub options of the freeform type. Note that the option # discovery doesn't care about the attribute name used here, so this # is just to avoid conflicts with potential options from the submodule From 27644a82a99b4855e40dfd6c09d7288664217662 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 29 Oct 2021 13:15:38 +0200 Subject: [PATCH 2/7] modules: Add extendModules to module args --- lib/modules.nix | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index c25972999dfe2..3a420451bf8e8 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -121,7 +121,9 @@ rec { }; config = { - _module.args = args; + _module.args = { + inherit extendModules; + } // args; }; }; @@ -186,24 +188,26 @@ rec { checked = builtins.seq checkUnmatched; + extendModules = extendArgs@{ + modules ? [], + specialArgs ? {}, + prefix ? [], + }: + evalModules (evalModulesArgs // { + modules = evalModulesArgs.modules ++ modules; + specialArgs = evalModulesArgs.specialArgs or {} // specialArgs; + prefix = extendArgs.prefix or evalModulesArgs.prefix; + }); + + type = lib.types.submoduleWith { + inherit modules specialArgs; + }; + result = { options = checked options; config = checked (removeAttrs config [ "_module" ]); _module = checked (config._module); - - extendModules = extendArgs@{ - modules ? [], - specialArgs ? {}, - prefix ? [], - }: - evalModules (evalModulesArgs // { - modules = evalModulesArgs.modules ++ modules; - specialArgs = evalModulesArgs.specialArgs or {} // specialArgs; - prefix = extendArgs.prefix or evalModulesArgs.prefix; - }); - type = lib.types.submoduleWith { - inherit modules specialArgs; - }; + inherit extendModules type; }; in result; From 64dfd983df61d9c427ac01d0d25b8d2f26b4c3d5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 28 Oct 2021 18:04:49 +0200 Subject: [PATCH 3/7] modules: Add visible = "shallow" to hide only sub-options --- lib/options.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/options.nix b/lib/options.nix index b3164181312ed..5d52f065af084 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -74,7 +74,7 @@ rec { apply ? null, # Whether the option is for NixOS developers only. internal ? null, - # Whether the option shows up in the manual. + # Whether the option shows up in the manual. Default: true. Use false to hide the option and any sub-options from submodules. Use "shallow" to hide only sub-options. visible ? null, # Whether the option can be set only once readOnly ? null, @@ -180,7 +180,10 @@ rec { description = opt.description or (lib.warn "Option `${name}' has no description." "This option has no description."); declarations = filter (x: x != unknownModule) opt.declarations; internal = opt.internal or false; - visible = opt.visible or true; + visible = + if (opt?visible && opt.visible == "shallow") + then true + else opt.visible or true; readOnly = opt.readOnly or false; type = opt.type.description or null; } @@ -192,8 +195,9 @@ rec { subOptions = let ss = opt.type.getSubOptions opt.loc; in if ss != {} then optionAttrSetToDocList' opt.loc ss else []; + subOptionsVisible = docOption.visible && opt.visible or null != "shallow"; in - [ docOption ] ++ optionals docOption.visible subOptions) (collect isOption options); + [ docOption ] ++ optionals subOptionsVisible subOptions) (collect isOption options); /* This function recursively removes all derivation attributes from From 22584ce6675fdabd14fd71f7ce8021cd14facf05 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 29 Oct 2021 13:40:22 +0200 Subject: [PATCH 4/7] nixos/eval-config.nix: Expose type Also works for (pkgs.nixos {}).type. --- nixos/lib/eval-config.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix index 15429a7160c5a..69e0a2afdba3d 100644 --- a/nixos/lib/eval-config.nix +++ b/nixos/lib/eval-config.nix @@ -61,7 +61,7 @@ in rec { args = extraArgs; specialArgs = { modulesPath = builtins.toString ../modules; } // specialArgs; - }) config options _module; + }) config options _module type; # These are the extra arguments passed to every module. In # particular, Nixpkgs is passed through the "pkgs" argument. From 86f5136bafec4a6208ffe23e870f74c731f849de Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 29 Oct 2021 14:58:28 +0200 Subject: [PATCH 5/7] modules: Update evalModules doc --- lib/modules.nix | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 3a420451bf8e8..d9b4000e56bdd 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -52,9 +52,32 @@ in rec { - /* Evaluate a set of modules. The result is a set of two - attributes: ‘options’: the nested set of all option declarations, - and ‘config’: the nested set of all option values. + /* + Evaluate a set of modules. The result is a set with the attributes: + + ‘options’: The nested set of all option declarations, + + ‘config’: The nested set of all option values. + + ‘type’: A module system type representing the module set as a submodule, + to be extended by configuration from the containing module set. + + ‘extendModules’: A function similar to ‘evalModules’ but building on top + of the module set. Its arguments, ‘modules’ and ‘specialArgs’ are + added to the existing values. + + Using ‘extendModules’ a few times has no performance impact as long + as you only reference the final ‘options’ and ‘config’. + If you do reference multiple ‘config’ (or ‘options’) from before and + after ‘extendModules’, performance is the same as with multiple + ‘evalModules’ invocations, because the new modules' ability to + override existing configuration fundamentally requires a new + fixpoint to be constructed. + + ‘_module’: A portion of the configuration tree which is elided from + ‘config’. It contains some values that are mostly internal to the + module system implementation. + !!! Please think twice before adding to this argument list! The more that is specified here instead of in the modules themselves the harder it is to transparently move a set of modules to be a submodule of another From e2bea4427bcf03ae168e45e0bcbd92ac678d91ce Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 29 Oct 2021 13:17:08 +0200 Subject: [PATCH 6/7] nixos/specialisation: Rephrase in terms of extendModules, noUserModules --- nixos/lib/eval-config.nix | 21 +++++++----- nixos/modules/system/activation/top-level.nix | 32 +++++++++++-------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix index 69e0a2afdba3d..74b52daa3c8eb 100644 --- a/nixos/lib/eval-config.nix +++ b/nixos/lib/eval-config.nix @@ -51,23 +51,28 @@ let }; }; -in rec { - - # Merge the option definitions in all modules, forming the full - # system configuration. - inherit (lib.evalModules { + noUserModules = lib.evalModules { inherit prefix check; - modules = baseModules ++ extraModules ++ [ pkgsModule ] ++ modules; + modules = baseModules ++ extraModules ++ [ pkgsModule ]; args = extraArgs; specialArgs = { modulesPath = builtins.toString ../modules; } // specialArgs; - }) config options _module type; + }; # These are the extra arguments passed to every module. In # particular, Nixpkgs is passed through the "pkgs" argument. extraArgs = extraArgs_ // { - inherit baseModules extraModules modules; + inherit noUserModules baseModules extraModules modules; }; +in rec { + + # Merge the option definitions in all modules, forming the full + # system configuration. + inherit (noUserModules.extendModules { inherit modules; }) + config options _module type; + + inherit extraArgs; + inherit (_module.args) pkgs; } diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 68da910d29cc8..8266622e78dfd 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, modules, baseModules, specialArgs, ... }: +{ config, lib, pkgs, extendModules, noUserModules, ... }: with lib; @@ -11,16 +11,10 @@ let # you can provide an easy way to boot the same configuration # as you use, but with another kernel # !!! fix this - children = mapAttrs (childName: childConfig: - (import ../../../lib/eval-config.nix { - inherit lib baseModules specialArgs; - system = config.nixpkgs.initialSystem; - modules = - (optionals childConfig.inheritParentConfig modules) - ++ [ ./no-clone.nix ] - ++ [ childConfig.configuration ]; - }).config.system.build.toplevel - ) config.specialisation; + children = + mapAttrs + (childName: childConfig: childConfig.configuration.system.build.toplevel) + config.specialisation; systemBuilder = let @@ -176,7 +170,11 @@ in ''; type = types.attrsOf (types.submodule ( - { ... }: { + local@{ ... }: let + extend = if local.config.inheritParentConfig + then extendModules + else noUserModules.extendModules; + in { options.inheritParentConfig = mkOption { type = types.bool; default = true; @@ -185,7 +183,15 @@ in options.configuration = mkOption { default = {}; - description = "Arbitrary NixOS configuration options."; + description = '' + Arbitrary NixOS configuration. + + Anything you can add to a normal NixOS configuration, you can add + here, including imports and config values, although nested + specialisations will be ignored. + ''; + visible = "shallow"; + inherit (extend { modules = [ ./no-clone.nix ]; }) type; }; }) ); From 0b0d2637fca96a4e4d7fac4c40e0773bc218ff53 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 1 Nov 2021 09:40:27 +0100 Subject: [PATCH 7/7] Revert "nixos/specialisation: Rephrase in terms of extendModules, noUserModules" This reverts commit e2bea4427bcf03ae168e45e0bcbd92ac678d91ce. While this commit was probably fine, I want to be conservative with changes right before the release branch-off. So far the extendModules commits have been adding and refactoring expressions that did not affect derivation hashes, whereas this commit changes the module ordering. I will open a separate PR for it. --- nixos/lib/eval-config.nix | 21 +++++------- nixos/modules/system/activation/top-level.nix | 32 ++++++++----------- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix index 74b52daa3c8eb..69e0a2afdba3d 100644 --- a/nixos/lib/eval-config.nix +++ b/nixos/lib/eval-config.nix @@ -51,28 +51,23 @@ let }; }; - noUserModules = lib.evalModules { +in rec { + + # Merge the option definitions in all modules, forming the full + # system configuration. + inherit (lib.evalModules { inherit prefix check; - modules = baseModules ++ extraModules ++ [ pkgsModule ]; + modules = baseModules ++ extraModules ++ [ pkgsModule ] ++ modules; args = extraArgs; specialArgs = { modulesPath = builtins.toString ../modules; } // specialArgs; - }; + }) config options _module type; # These are the extra arguments passed to every module. In # particular, Nixpkgs is passed through the "pkgs" argument. extraArgs = extraArgs_ // { - inherit noUserModules baseModules extraModules modules; + inherit baseModules extraModules modules; }; -in rec { - - # Merge the option definitions in all modules, forming the full - # system configuration. - inherit (noUserModules.extendModules { inherit modules; }) - config options _module type; - - inherit extraArgs; - inherit (_module.args) pkgs; } diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 8266622e78dfd..68da910d29cc8 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, extendModules, noUserModules, ... }: +{ config, lib, pkgs, modules, baseModules, specialArgs, ... }: with lib; @@ -11,10 +11,16 @@ let # you can provide an easy way to boot the same configuration # as you use, but with another kernel # !!! fix this - children = - mapAttrs - (childName: childConfig: childConfig.configuration.system.build.toplevel) - config.specialisation; + children = mapAttrs (childName: childConfig: + (import ../../../lib/eval-config.nix { + inherit lib baseModules specialArgs; + system = config.nixpkgs.initialSystem; + modules = + (optionals childConfig.inheritParentConfig modules) + ++ [ ./no-clone.nix ] + ++ [ childConfig.configuration ]; + }).config.system.build.toplevel + ) config.specialisation; systemBuilder = let @@ -170,11 +176,7 @@ in ''; type = types.attrsOf (types.submodule ( - local@{ ... }: let - extend = if local.config.inheritParentConfig - then extendModules - else noUserModules.extendModules; - in { + { ... }: { options.inheritParentConfig = mkOption { type = types.bool; default = true; @@ -183,15 +185,7 @@ in options.configuration = mkOption { default = {}; - description = '' - Arbitrary NixOS configuration. - - Anything you can add to a normal NixOS configuration, you can add - here, including imports and config values, although nested - specialisations will be ignored. - ''; - visible = "shallow"; - inherit (extend { modules = [ ./no-clone.nix ]; }) type; + description = "Arbitrary NixOS configuration options."; }; }) );