What about modules to define modules?

[hugosenari]

I think people don't declare modules enough.
Maybe because declare a module (options) isn't fun¹.

What about modules to define modules?

¹ https://www.youtube.com/watch?v=dTd499Y31ig

[DavHau]

Oh thanks for reminding me of this amazing talk which likely had originally inspired me thinking about all this.

What about modules to define modules?

Hm, sounds interesting. Would you mind providing an example of how you imagine this to look like?

[hugosenari]

Oh thanks for reminding me of this amazing talk which likely had originally inspired me thinking about all this.

I think Nix community didn't watch it enough, but it ends towards Nickel like solution, while we already have tools to make it nice.

Hm, sounds interesting. Would you mind providing an example of how you imagine this to look like?

First, I'm not expert in modules, but I realize that we divide it in two, where it could be divided in three.

Normal user: Thinking in modules as 1 file (config)
Mid-user: Thinking in modules as 1 file (config and options)
Advanced user: Thinking in modules as 2 files (config xor options)
My suggestion: Thinking in modules as 3 files (config xor options xor implementation)

config values (declarative)
options define expected interfaces, like C .h (declarative)
implementation define what to with values, like C .c files (imperative)

The nearest I have is this, but would be good if I haven't to call any function and what inside let to inside a { }

[hugosenari]

What if instead of this:

{ lib, ...}:
let
  Node = lib.types.submodule {
    options.tags = lib.mkOption {
      type        = lib.types.listOf lib.types.str;
      example     = ["logstash"];
      default     = [];
      description = "Define the tags of this node in the project cluster";
    };
  };
  env = lib.mkOption {
    type        = lib.types.attrsOf Node;
    default     = {};
    example     = { logstash01 = { tags = ["logstash"]; }; };
    description = "nodes of this environment";
  };

  Project = lib.types.submodule {
    options.git  = lib.mkOption {
      default     = "";
      type        = lib.types.str;
      example     = "https://github.com/DavHau";
      description = "git project url";
    };
    options.desc = lib.mkOption {
      default     = "";
      type        = lib.types.str;
      example     = "Elastic stack config files";
      description = "Description of the project";
    };
    options.docs = lib.mkOption {
      default     = "";
      type        = lib.types.str;
      example     = "https://google.com";
      description = "url of main documentations";
    };
    options.dev = env;
    options.prd = env;
  };
in
{
  options.project = lib.mkOption {
    type        = lib.types.attrsOf Project;
    default     = {};
    example     = { elk.desc = "elastic stack"; elk.dev.logstash01.tags = ["logstash"]; };
    description = "Information about our git projects";
  };
}

We do this:

{lib, ...}:
let
  env.default      = {};
  env.example      = { logstash01 = { tags = ["logstash"]; }; };
  env.description  = "nodes of this environment";
  env."<name>".options.tags."*".type        = lib.types.str;
  env."<name>".options.tags.example     = ["logstash"];
  env."<name>".options.tags.default     = [];
  env."<name>".options.tags.description = "Define the tags of this node in the project cluster";
in
{
  options.project.default     = {};
  options.project.example     = { elk.desc = "elastic stack"; elk.dev.logstash01.tags = ["logstash"]; };
  options.project.description = "Information about our git projects";
  options.project."<name>".options.git.default      = "";
  options.project."<name>".options.git.type         = lib.types.str;
  options.project."<name>".options.git.example      = "https://github.com/DavHau/pkgs-modules";
  options.project."<name>".options.git.description  = "git project url";
  options.project."<name>".options.desc.default     = "";
  options.project."<name>".options.desc.type        = lib.types.str;
  options.project."<name>".options.desc.example     = "Elastic stack config files";
  options.project."<name>".options.desc.description = "Description of the project";
  options.project."<name>".options.docs.default     = "";
  options.project."<name>".options.docs.type        = lib.types.str;
  options.project."<name>".options.docs.example     = "https://google.com";
  options.project."<name>".options.docs.description = "url of main documentations";
  options.project."<name>".options.dev = env;
  options.project."<name>".options.prd = env;
}

Then we recurse over this as:
if it has options, is a submodule
if it has <name>, is attrOf
if it has *, is listOf
or else type is required

[DavHau]

At first glance this looks pretty cool.
Maybe some syntactic sugar like this could be upstreamed to nixpkgs/lib/modules.nix

What do you think about this style proposed in the previous comment @roberth ?

[roberth]

I think the module system already has too much syntax sugar. Every time you add syntax sugar, you introduce at least one corner case. In your example, each mkOption parameter becomes a "keyword" or a possible name collision.
Furthermore, you obfuscate the original data model. You might flatten the very start of the learning curve, but a newcomer will have learned the wrong thing, and still needs to learn the original syntax before they become proficient.

  options.project."<name>".options.git.example      = "https://github.com/DavHau/pkgs-modules";
  options.project."<name>".options.git.description  = "git project url";
  options.project."<name>".options.desc.default     = "";
  options.project."<name>".options.desc.type        = lib.types.str;
  options.project."<name>".options.desc.example     = "Elastic stack config files";

I find this quite unpleasant to read. Nesting the attrset syntax might improve it for me, but at that point you've only removed the need for mkOption, and you've replaced attrsOf by <name>.
Imagine the horror when they inevitably write a project.foo that references project.bar and they have to rewrite the whole options.project declaration with lazyAttrsOf.
Similarly for defining submodule-local logic.

Instead of adding more sugar, I'd prefer a simplification, such as turning the options into an "attrsOf option" rather than a tree. This also removes the need for mkOption, but without introducing name collisions.

[roberth]

Advanced user: Thinking in modules as 2 files (config xor options)

It's rare for the options not to come with default values. I don't find a separation into files useful, but I guess it could serve as a mental model. I wouldn't call them files though, but something more abstract in that case.

Thinking in modules as 3 files (config xor options xor implementation)

I don't distinguish between implementation and config.
What sets implementation apart?
Maybe hiding intermediate values in let bindings? I find that exposing intermediate values as options is a good practice. Often they seem unimportant at first, but later you want to read or override them.

[hugosenari]

You might flatten the very start of the learning curve.

Good old joke

The usage of an existing module is nice and easy, but the creation of a module is a fine art that requires a lot of fu. Part of Eelco video about creation of an module 'lang' is that the type definition isn't easy, and maybe is why we have Nickel in progress that may fix the issue, but isn't declarative/homoiconic as nix module.

And we may have some of extraOptions because the pains of type definition. Like "i would like to properly type my options, but nah.. too much work. just add extraOptions".

That is not to say that my suggestions are good, but simplifying type definition is (even with other methods).

"attrsOf option"

This is a function call, this project suggest using module system instead of mkDerivation directly because is declarative instead of imperative and that impacts readability and modularity. To be honest, "*".type = str in my example would be less readable than listOf str, but wouldn't if it was as nested type. The idea is to have less (not never) function calls. To this particular example listOf stris recomended, but I need some example of listOf and I'm too lazy (this is a real example) to create one with nested types now.

I think the module system already has too much syntax sugar.

Mixing this quote, previous one and the next. Last week I discovered. tweag/nix-hour#15 that surprised me so much, that I tried to actually implement it in different files, extending my previous example (as I told this is a real example).
And turns out that I should have to write this in another file:

{ lib, ...}:
let
  SSH = lib.types.submodule {
    options.host = lib.mkOption {
      default     = "";
      type        = lib.types.str;
      example     = "127.0.0.1";
      description = "SSH host";
    };
    options.user = lib.mkOption {
      default     = "";
      type        = lib.types.str;
      example     = "batatinha";
      description = "SSH user";
    };
  };
  ssh = lib.mkOption {
    default     = {};
    example     = { host = "127.0.0.1"; user = "batatinha"; };
    description = "ssh options";
    type        = SSH;
  };
  Node    = lib.types.submodule { options.ssh = ssh; };
  env     = lib.mkOption        { type = lib.types.attrsOf Node; };
  Project = lib.types.submodule {
    options.dev = env;
    options.prd = env;
  };
in {
  options.project = lib.mkOption {
    type = lib.types.attrsOf Project;
  };

but could be

let 
  env."<name>".options.ssh = {
    default     = {};
    example     = { host = "127.0.0.1"; user = "batatinha"; };
    description = "ssh options";
    options = {
      user.default     = "";
      user.type        = lib.types.str;
      user.example     = "batatinha";
      user.description = "SSH user";
      host.default     = "";
      host.type        = lib.types.str;
      host.example     = "127.0.0.1";
      host.description = "SSH host";
   };
};
in {
  options.project."<name>".options.dev = env;
  options.project."<name>".options.prd = env;
}

I find this quite unpleasant to read.

Thanks, your feedback is appreciated, I have a very strange taste, you may see it by my indentation and repetition. Maybe my editor makes easier to copy lines than indent lines. Or because of my java properties background, I think modules like java properties are easy to any OPs reason about than our with, inherit, let/in and (LISP (function (calls (wih args))).

but at that point you've only removed the need for mkOption, and you've replaced attrsOf by .

As most of current config is some sort of 'json', I think 'attrsOf = .<name>' (dict), 'submodule = .options' (object) and 'listOf = .*' (array), with primitive types, cover most user cases (citation needed). SetOf would be the next, but how... 🤔

Furthermore, you obfuscate the original data model.
newcomer will have learned the wrong thing

I suggested those (<name>, *) thinking in 'definition <=> doc' parity, based on current doc tools, so that means, our current doc tools are wrong? I would like remove the options part to better match docs but I don't know how.
Perhaps, would make sense choose something more friendly to newcorners, ie <name> to be <key>, since 'key/value' is well known concept. 🤔

introducing name collisions

It would only makes sense as default behavior after a long term as lib and if gains popularity. So yes, name collisions but as opt in.

It's rare for the options not to come with default values.
What sets implementation apart?

The best example for that is: cruel-intentions/devshell-files#8 (devshell files without devshell, because we have devenv, and others) and cachix/devenv#75 (make services usable outside devenv), like make service definition usable in other init systems, describing both options and how the modules converts it to systemd services files, would be more reusable if options are in one file without any 'config' (except defaults).

But in this case is more because while we can declarative describe options (interface), I can't say the same of how we convert that to final format/"config"/file/derivation (implementation). That, sadly, would not make easier to convert expected type (interface) to final form (data wrangling).

My current implementation isn't working (yet), but I think is good to have here for curiosity.

{ lib, ...}: opt-def:
let 
  attrs = attrSet: builtins.concatStringsSep ", " (builtins.attrNames attrSet);
  split = path: opt-def': 
  let prop = 
    if builtins.hasAttr "type"    opt-def' then "type"
    else
    if builtins.hasAttr "_type"   opt-def' then "_type"
    else
    if builtins.hasAttr "options" opt-def' then "options" 
    else
    if builtins.hasAttr "<name>"  opt-def' then "<name>"
    else
    if builtins.hasAttr "*"       opt-def' then "*"
    else
    builtins.throw ''
      Option type def error, we don't known what to do with this
      Expected ${path} with one of these attr "type", "options", "<name>", "*" but got
      ${attrs opt-def'}
    '';
  in {
    inherit prop;
    type = opt-def'."${prop}" or null;
    mod  = builtins.removeAttrs opt-def' [prop];
  };
  toSub = path: opt-def': { options = builtins.mapAttrs (prop: sub: lib.mkOption (toOpt "${path}.${prop}" sub)) opt-def'; };
  toOpt = path: opt-def':
  let 
    splited = split path opt-def';
    ifHas._type    = { prop, type, mod }: opt-def';
    ifHas.type     = { prop, type, mod }: mod // { type = type; };
    ifHas."<name>" = { prop, type, mod }: mod // { type = lib.types.attrsOf   (toOpt "${path}.${prop}" type); };
    ifHas."*"      = { prop, type, mod }: mod // { type = lib.types.listOf    (toOpt "${path}.${prop}" type); };
    ifHas.options  = { prop, type, mod }: mod // { type = lib.types.submodule (toSub "${path}.${prop}" type); };
    impl = ifHas."${splited.prop}";
  in builtins.trace "${path} ${splited.prop} (${attrs (impl splited)})" impl splited;
in toSub "options" opt-def

# should be called like
# let
#   sadnessAndSorrow = import ./im_bad_at_naming_things.nix { inherit lib; };
#   # isn't working, is an appropriate soundtrack
# in {
#   imports = [ (sadnessAndSorrow ./projects.nix) (sadnessAndSorrow ./ssh.nix) ];
# }

[roberth]

Maybe 1% better:

{ lib, ... }:
lib.simpleModule {
  options = .....;
  config = .....;
}

Still not sure if it's a good idea, because people will try to use this for as long as possible, even if they really shouldn't because they need all sorts of workarounds. Aside from having to learn more unnecessary syntax.
Brevity is nice until you're stuck. I don't think we want to treat users that way.

[hugosenari]

Now we have a POC.

I tried solve collision problem by adding parameters to let user define the keywords.

I tried validation of attributes but module system, give some strange errors when IE. someone type attrOf instead of attrsOf :-(