diff --git a/lib/generators.nix b/lib/generators.nix
index efe6ea6031d33..abd237eb7d377 100644
--- a/lib/generators.nix
+++ b/lib/generators.nix
@@ -48,8 +48,10 @@ rec {
else if isAttrs v then err "attrsets" v
# functions can’t be printed of course
else if isFunction v then err "functions" v
- # let’s not talk about floats. There is no sensible `toString` for them.
- else if isFloat v then err "floats" v
+ # Floats currently can't be converted to precise strings,
+ # condition warning on nix version once this isn't a problem anymore
+ # See https://github.com/NixOS/nix/pull/3480
+ else if isFloat v then libStr.floatToString v
else err "this value is" (toString v);
diff --git a/lib/strings.nix b/lib/strings.nix
index 74e3eaa0722de..0baa942355c0e 100644
--- a/lib/strings.nix
+++ b/lib/strings.nix
@@ -612,6 +612,22 @@ rec {
*/
fixedWidthNumber = width: n: fixedWidthString width "0" (toString n);
+ /* Convert a float to a string, but emit a warning when precision is lost
+ during the conversion
+
+ Example:
+ floatToString 0.000001
+ => "0.000001"
+ floatToString 0.0000001
+ => trace: warning: Imprecise conversion from float to string 0.000000
+ "0.000000"
+ */
+ floatToString = float: let
+ result = toString float;
+ precise = float == builtins.fromJSON result;
+ in if precise then result
+ else lib.warn "Imprecise conversion from float to string ${result}" result;
+
/* Check whether a value can be coerced to a string */
isCoercibleToString = x:
builtins.elem (builtins.typeOf x) [ "path" "string" "null" "int" "float" "bool" ] ||
diff --git a/nixos/doc/manual/development/settings-options.xml b/nixos/doc/manual/development/settings-options.xml
new file mode 100644
index 0000000000000..84895adb444d3
--- /dev/null
+++ b/nixos/doc/manual/development/settings-options.xml
@@ -0,0 +1,179 @@
+
+ Options for Program Settings
+
+
+ Many programs have configuration files where program-specific settings can be declared. File formats can be separated into two categories:
+
+
+
+ Nix-representable ones: These can trivially be mapped to a subset of Nix syntax. E.g. JSON is an example, since its values like {"foo":{"bar":10}} can be mapped directly to Nix: { foo = { bar = 10; }; }. Other examples are INI, YAML and TOML. The following section explains the convention for these settings.
+
+
+
+
+ Non-nix-representable ones: These can't be trivially mapped to a subset of Nix syntax. Most generic programming languages are in this group, e.g. bash, since the statement if true; then echo hi; fi doesn't have a trivial representation in Nix.
+
+
+ Currently there are no fixed conventions for these, but it is common to have a configFile option for setting the configuration file path directly. The default value of configFile can be an auto-generated file, with convenient options for controlling the contents. For example an option of type attrsOf str can be used for representing environment variables which generates a section like export FOO="foo". Often it can also be useful to also include an extraConfig option of type lines to allow arbitrary text after the autogenerated part of the file.
+
+
+
+
+
+ Nix-representable Formats (JSON, YAML, TOML, INI, ...)
+
+ By convention, formats like this are handled with a generic settings option, representing the full program configuration as a Nix value. The type of this option should represent the format. The most common formats have a predefined type and string generator already declared under pkgs.formats:
+
+
+
+ pkgs.formats.json { }
+
+
+
+ A function taking an empty attribute set (for future extensibility) and returning a set with JSON-specific attributes type and generate as specified below.
+
+
+
+
+
+ pkgs.formats.yaml { }
+
+
+
+ A function taking an empty attribute set (for future extensibility) and returning a set with YAML-specific attributes type and generate as specified below.
+
+
+
+
+
+ pkgs.formats.ini { listsAsDuplicateKeys ? false, ... }
+
+
+
+ A function taking an attribute set with values
+
+
+
+ listsAsDuplicateKeys
+
+
+
+ A boolean for controlling whether list values can be used to represent duplicate INI keys
+
+
+
+
+ It returns a set with INI-specific attributes type and generate as specified below.
+
+
+
+
+
+ pkgs.formats.toml { }
+
+
+
+ A function taking an empty attribute set (for future extensibility) and returning a set with TOML-specific attributes type and generate as specified below.
+
+
+
+
+
+
+
+ These functions all return an attribute set with these values:
+
+
+
+ type
+
+
+
+ A module system type representing a value of the format
+
+
+
+
+
+ generatefilenamejsonValue
+
+
+
+ A function that can render a value of the format to a file. Returns a file path.
+
+
+ This function puts the value contents in the Nix store. So this should be avoided for secrets.
+
+
+
+
+
+
+
+
+ Module with conventional <literal>settings</literal> option
+
+ The following shows a module for an example program that uses a JSON configuration file. It demonstrates how above values can be used, along with some other related best practices. See the comments for explanations.
+
+
+{ options, config, lib, pkgs, ... }:
+let
+ cfg = config.services.foo;
+ # Define the settings format used for this program
+ settingsFormat = pkgs.formats.json {};
+in {
+
+ options.services.foo = {
+ enable = lib.mkEnableOption "foo service";
+
+ settings = lib.mkOption {
+ # Setting this type allows for correct merging behavior
+ type = settingsFormat.type;
+ default = {};
+ description = ''
+ Configuration for foo, see
+ <link xlink:href="https://example.com/docs/foo"/>
+ for supported values.
+ '';
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ # We can assign some default settings here to make the service work by just
+ # enabling it. We use `mkDefault` for values that can be changed without
+ # problems
+ services.foo.settings = {
+ # Fails at runtime without any value set
+ log_level = lib.mkDefault "WARN";
+
+ # We assume systemd's `StateDirectory` is used, so we require this value,
+ # therefore no mkDefault
+ data_path = "/var/lib/foo";
+
+ # Since we use this to create a user we need to know the default value at
+ # eval time
+ user = lib.mkDefault "foo";
+ };
+
+ environment.etc."foo.json".source =
+ # The formats generator function takes a filename and the Nix value
+ # representing the format value and produces a filepath with that value
+ # rendered in the format
+ settingsFormat.generate "foo-config.json" cfg.settings;
+
+ # We know that the `user` attribute exists because we set a default value
+ # for it above, allowing us to use it without worries here
+ users.users.${cfg.settings.user} = {}
+
+ # ...
+ };
+}
+
+
+
+
+
diff --git a/nixos/doc/manual/development/writing-modules.xml b/nixos/doc/manual/development/writing-modules.xml
index bbf793bb0be98..602f134f9cbfa 100644
--- a/nixos/doc/manual/development/writing-modules.xml
+++ b/nixos/doc/manual/development/writing-modules.xml
@@ -183,4 +183,5 @@ in {
+
diff --git a/pkgs/pkgs-lib/default.nix b/pkgs/pkgs-lib/default.nix
new file mode 100644
index 0000000000000..113dcebf8c68b
--- /dev/null
+++ b/pkgs/pkgs-lib/default.nix
@@ -0,0 +1,11 @@
+# pkgs-lib is for functions and values that can't be in lib because
+# they depend on some packages. This notably is *not* for supporting package
+# building, instead pkgs/build-support is the place for that.
+{ lib, pkgs }: {
+ # setting format types and generators. These do not fit in lib/types.nix,
+ # because they depend on pkgs for rendering some formats
+ formats = import ./formats.nix {
+ inherit lib pkgs;
+ };
+}
+
diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix
new file mode 100644
index 0000000000000..14589f8ecdc3a
--- /dev/null
+++ b/pkgs/pkgs-lib/formats.nix
@@ -0,0 +1,109 @@
+{ lib, pkgs }:
+rec {
+
+ /*
+
+ Every following entry represents a format for program configuration files
+ used for `settings`-style options (see https://github.com/NixOS/rfcs/pull/42).
+ Each entry should look as follows:
+
+ = : {
+ # ^^ Parameters for controlling the format
+
+ # The module system type most suitable for representing such a format
+ # The description needs to be overwritten for recursive types
+ type = ...;
+
+ # generate :: Name -> Value -> Path
+ # A function for generating a file with a value of such a type
+ generate = ...;
+
+ });
+ */
+
+
+ json = {}: {
+
+ type = with lib.types; let
+ valueType = nullOr (oneOf [
+ bool
+ int
+ float
+ str
+ (attrsOf valueType)
+ (listOf valueType)
+ ]) // {
+ description = "JSON value";
+ };
+ in valueType;
+
+ generate = name: value: pkgs.runCommandNoCC name {
+ nativeBuildInputs = [ pkgs.jq ];
+ value = builtins.toJSON value;
+ passAsFile = [ "value" ];
+ } ''
+ jq . "$valuePath"> $out
+ '';
+
+ };
+
+ # YAML has been a strict superset of JSON since 1.2
+ yaml = {}:
+ let jsonSet = json {};
+ in jsonSet // {
+ type = jsonSet.type // {
+ description = "YAML value";
+ };
+ };
+
+ ini = { listsAsDuplicateKeys ? false, ... }@args: {
+
+ type = with lib.types; let
+
+ singleIniAtom = nullOr (oneOf [
+ bool
+ int
+ float
+ str
+ ]) // {
+ description = "INI atom (null, bool, int, float or string)";
+ };
+
+ iniAtom =
+ if listsAsDuplicateKeys then
+ coercedTo singleIniAtom lib.singleton (listOf singleIniAtom) // {
+ description = singleIniAtom.description + " or a list of them for duplicate keys";
+ }
+ else
+ singleIniAtom;
+
+ in attrsOf (attrsOf iniAtom);
+
+ generate = name: value: pkgs.writeText name (lib.generators.toINI args value);
+
+ };
+
+ toml = {}: json {} // {
+ type = with lib.types; let
+ valueType = oneOf [
+ bool
+ int
+ float
+ str
+ (attrsOf valueType)
+ (listOf valueType)
+ ] // {
+ description = "TOML value";
+ };
+ in valueType;
+
+ generate = name: value: pkgs.runCommandNoCC name {
+ nativeBuildInputs = [ pkgs.remarshal ];
+ value = builtins.toJSON value;
+ passAsFile = [ "value" ];
+ } ''
+ json2toml "$valuePath" "$out"
+ '';
+
+ };
+}
diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix
new file mode 100644
index 0000000000000..f3549ea9b0f2c
--- /dev/null
+++ b/pkgs/pkgs-lib/tests/default.nix
@@ -0,0 +1,7 @@
+# Call nix-build on this file to run all tests in this directory
+{ pkgs ? import ../../.. {} }:
+let
+ formats = import ./formats.nix { inherit pkgs; };
+in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [
+ { name = "formats"; path = import ./formats.nix { inherit pkgs; }; }
+]
diff --git a/pkgs/pkgs-lib/tests/formats.nix b/pkgs/pkgs-lib/tests/formats.nix
new file mode 100644
index 0000000000000..bf6be8595e1ba
--- /dev/null
+++ b/pkgs/pkgs-lib/tests/formats.nix
@@ -0,0 +1,157 @@
+{ pkgs }:
+let
+ inherit (pkgs) lib formats;
+in
+with lib;
+let
+
+ evalFormat = format: args: def:
+ let
+ formatSet = format args;
+ config = formatSet.type.merge [] (imap1 (n: def: {
+ value = def;
+ file = "def${toString n}";
+ }) [ def ]);
+ in formatSet.generate "test-format-file" config;
+
+ runBuildTest = name: { drv, expected }: pkgs.runCommandNoCC name {} ''
+ if diff ${drv} ${builtins.toFile "expected" expected}; then
+ touch $out
+ else
+ echo "Got: $(cat ${drv})"
+ echo "Should be: ${expected}"
+ exit 1
+ fi
+ '';
+
+ runBuildTests = tests: pkgs.linkFarm "nixpkgs-pkgs-lib-format-tests" (mapAttrsToList (name: value: { inherit name; path = runBuildTest name value; }) (filterAttrs (name: value: value != null) tests));
+
+in runBuildTests {
+
+ testJsonAtoms = {
+ drv = evalFormat formats.json {} {
+ null = null;
+ false = false;
+ true = true;
+ int = 10;
+ float = 3.141;
+ str = "foo";
+ attrs.foo = null;
+ list = [ null null ];
+ };
+ expected = ''
+ {
+ "attrs": {
+ "foo": null
+ },
+ "false": false,
+ "float": 3.141,
+ "int": 10,
+ "list": [
+ null,
+ null
+ ],
+ "null": null,
+ "str": "foo",
+ "true": true
+ }
+ '';
+ };
+
+ testYamlAtoms = {
+ drv = evalFormat formats.yaml {} {
+ null = null;
+ false = false;
+ true = true;
+ float = 3.141;
+ str = "foo";
+ attrs.foo = null;
+ list = [ null null ];
+ };
+ expected = ''
+ {
+ "attrs": {
+ "foo": null
+ },
+ "false": false,
+ "float": 3.141,
+ "list": [
+ null,
+ null
+ ],
+ "null": null,
+ "str": "foo",
+ "true": true
+ }
+ '';
+ };
+
+ testIniAtoms = {
+ drv = evalFormat formats.ini {} {
+ foo = {
+ bool = true;
+ int = 10;
+ float = 3.141;
+ str = "string";
+ };
+ };
+ expected = ''
+ [foo]
+ bool=true
+ float=3.141000
+ int=10
+ str=string
+ '';
+ };
+
+ testIniDuplicateKeys = {
+ drv = evalFormat formats.ini { listsAsDuplicateKeys = true; } {
+ foo = {
+ bar = [ null true "test" 1.2 10 ];
+ baz = false;
+ qux = "qux";
+ };
+ };
+ expected = ''
+ [foo]
+ bar=null
+ bar=true
+ bar=test
+ bar=1.200000
+ bar=10
+ baz=false
+ qux=qux
+ '';
+ };
+
+ testTomlAtoms = {
+ drv = evalFormat formats.toml {} {
+ false = false;
+ true = true;
+ int = 10;
+ float = 3.141;
+ str = "foo";
+ attrs.foo = "foo";
+ list = [ 1 2 ];
+ level1.level2.level3.level4 = "deep";
+ };
+ expected = ''
+ false = false
+ float = 3.141
+ int = 10
+ list = [1, 2]
+ str = "foo"
+ true = true
+
+ [attrs]
+ foo = "foo"
+
+ [level1]
+
+ [level1.level2]
+
+ [level1.level2.level3]
+ level4 = "deep"
+ '';
+ };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 6427bb4bbc261..07e5fca168500 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -523,6 +523,9 @@ in
#package writers
writers = callPackage ../build-support/writers {};
+ # lib functions depending on pkgs
+ inherit (import ../pkgs-lib { inherit lib pkgs; }) formats;
+
### TOOLS
_0x0 = callPackage ../tools/misc/0x0 { };
diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix
index c11858f09c8f5..5fc6e91b31181 100644
--- a/pkgs/top-level/release.nix
+++ b/pkgs/top-level/release.nix
@@ -34,6 +34,7 @@ let
manual = import ../../doc { inherit pkgs nixpkgs; };
lib-tests = import ../../lib/tests/release.nix { inherit pkgs; };
+ pkgs-lib-tests = import ../pkgs-lib/tests { inherit pkgs; };
darwin-tested = if supportDarwin then pkgs.releaseTools.aggregate
{ name = "nixpkgs-darwin-${jobs.tarball.version}";
@@ -91,6 +92,7 @@ let
[ jobs.tarball
jobs.manual
jobs.lib-tests
+ jobs.pkgs-lib-tests
jobs.stdenv.x86_64-linux
jobs.linux.x86_64-linux
jobs.pandoc.x86_64-linux