tree-sitter: add schema to grammar definition, support multi-grammar repos

pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix

@@ -4,7 +4,7 @@ self: super:
 
 let
   generatedGrammars = callPackage ./generated.nix {
-    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/grammar.nix { };
+    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/build-grammar.nix { };
   };
 
   generatedDerivations = lib.filterAttrs (_: lib.isDerivation) generatedGrammars;

pkgs/development/tools/parsing/tree-sitter/README.md

@@ -0,0 +1,41 @@
+# tree-sitter libraries, binaries & grammars
+
+This packages tree sitter and its grammars.
+
+The grammar descriptions can be found in [./grammars.toml]().
+
+## Updating tree-sitter
+
+1) change all hashes at the beginning of [./default.nix]().
+2) Update the grammars (see below)
+
+## Updating all grammars
+
+First you need a github Personal Access Token, otherwise it runs into rate limits.
+Go to https://github.com/settings/tokens and generate a classic token, copy the secret.
+
+You generate the update script and run it:
+
+```bash
+$ nix-build -A tree-sitter.updater.update-all-grammars
+$ env GITHUB_TOKEN=<secret token> ./result
+```
+
+This will prefetch all repos mentioned in [./grammars.toml]() and put their new hashes
+into the [./grammars]() directory.
+
+If a new repository was added to the `github.com/tree-sitter` organization,
+the update process will throw an error and you need to add the new repo to
+either `knownTreeSitterOrgGrammarRepos` (if it’s a grammar) or to
+`ignoredTreeSitterOrgRepos`.
+This is to make sure we always package every official grammar.
+
+## Adding a third-party grammar
+
+Add it to the `otherGrammars` section in [./grammars.toml]().
+The grammar name has to be unique among all grammars (upstream and third party).
+
+## Deleting a grammar
+
+In case a grammar needs to be removed, please remove the generated outputs
+in the [./grammar]() directory manually.

pkgs/development/tools/parsing/tree-sitter/grammar.nix → pkgs/development/tools/parsing/tree-sitter/build-grammar.nix

@@ -12,6 +12,7 @@
 , version
   # source for the language grammar
 , source
+  # subdirectory inside of source that contains the actual grammar, or `null` of none.
 , location ? null
 }:
 
@@ -20,7 +21,10 @@ stdenv.mkDerivation rec {
   pname = "${language}-grammar";
   inherit version;
 
-  src = if location == null then source else "${source}/${location}";
+  src =
+    if location == null
+    then source
+    else "${source}/${location}";
 
   buildInputs = [ tree-sitter ];
 

pkgs/development/tools/parsing/tree-sitter/default.nix

@@ -20,16 +20,15 @@
 , enableShared ? !stdenv.hostPlatform.isStatic
 , enableStatic ? stdenv.hostPlatform.isStatic
 , webUISupport ? false
+
+# REMOVED
 , extraGrammars ? { }
 }:
 
-# TODO: move to carnix or https://github.com/kolloch/crate2nix
+assert lib.assertMsg (extraGrammars == {}) "The `extraGrammars` for tree-sitter was removed, because the schema of extraGrammars was underspecified & undocumented. If you need support, please open an issue and ping @Profpatsch";
+
 let
-  # to update:
-  # 1) change all these hashes
-  # 2) nix-build -A tree-sitter.updater.update-all-grammars
-  # 3) Set GITHUB_TOKEN env variable to avoid api rate limit (Use a Personal Access Token from https://github.com/settings/tokens It does not need any permissions)
-  # 4) run the ./result script that is output by that (it updates ./grammars)
+  # to update: see ./README.md
   version = "0.20.7";
   sha256 = "sha256-5ILiN5EfJ7WpeYBiXynfcLucdp8zmxVOj4gLkaFQYts=";
   cargoSha256 = "sha256-V4frCaU5QzTx3ujdaplw7vNkosbzyXHQvE+T7ntVOtU=";
@@ -46,30 +45,38 @@ let
 
   fetchGrammar = (v: fetchgit { inherit (v) url rev sha256 fetchSubmodules; });
 
+  # TODO: use linkFarm
+
+  # All grammar definitions’ source repositories.
   grammars =
-    runCommand "grammars" { } (''
+    runCommand "grammars" { } ''
       mkdir $out
-    '' + (lib.concatStrings (lib.mapAttrsToList
-      (name: grammar: "ln -s ${if grammar ? src then grammar.src else fetchGrammar grammar} $out/${name}\n")
-      (import ./grammars { inherit lib; }))));
+      ${lib.pipe
+         (import ./grammars { inherit lib; })
+         [
+          (lib.mapAttrsToList
+            (name: grammar: "ln -s ${fetchGrammar grammar} $out/${name}\n"))
+          lib.concatStrings
+         ]}
+    '';
+
   builtGrammars =
     let
       change = name: grammar:
-        callPackage ./grammar.nix { } {
-          language = if grammar ? language then grammar.language else name;
+        callPackage ./build-grammar.nix { } {
+          language = name;
           inherit version;
-          source = if grammar ? src then grammar.src else fetchGrammar grammar;
+          source = fetchGrammar grammar;
           location = if grammar ? location then grammar.location else null;
         };
-      grammars' = import ./grammars { inherit lib; } // extraGrammars;
+      grammars' = import ./grammars { inherit lib; };
       grammars = grammars' //
         { tree-sitter-ocaml = grammars'.tree-sitter-ocaml // { location = "ocaml"; }; } //
         { tree-sitter-ocaml-interface = grammars'.tree-sitter-ocaml // { location = "interface"; }; } //
-        { tree-sitter-org-nvim = grammars'.tree-sitter-org-nvim // { language = "org"; }; } //
         { tree-sitter-typescript = grammars'.tree-sitter-typescript // { location = "typescript"; }; } //
         { tree-sitter-tsx = grammars'.tree-sitter-typescript // { location = "tsx"; }; } //
         { tree-sitter-markdown = grammars'.tree-sitter-markdown // { location = "tree-sitter-markdown"; }; } //
-        { tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // { language = "markdown_inline"; location = "tree-sitter-markdown-inline"; }; };
+        { tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // { location = "tree-sitter-markdown-inline"; }; };
     in
     lib.mapAttrs change (grammars);
 
@@ -81,25 +88,23 @@ let
   # which is equivalent to
   # pkgs.tree-sitter.withPlugins (p: builtins.attrValues p)
   withPlugins = grammarFn:
-    let
-      grammars = grammarFn builtGrammars;
-    in
-    linkFarm "grammars"
+    lib.pipe builtGrammars [
+      grammarFn
       (map
         (drv:
-          let
-            name = lib.strings.getName drv;
-          in
           {
-            name =
-              (lib.strings.replaceStrings [ "-" ] [ "_" ]
-                (lib.strings.removePrefix "tree-sitter-"
-                  (lib.strings.removeSuffix "-grammar" name)))
-              + ".so";
+            name = lib.pipe drv [
+              lib.strings.getName
+              (lib.strings.removeSuffix "-grammar")
+              (lib.strings.removePrefix "tree-sitter-")
+              (lib.strings.replaceStrings [ "-" ] [ "_" ])
+              (name: name + ".so")
+            ];
             path = "${drv}/parser";
           }
-        )
-        grammars);
+        ))
+      (linkFarm "grammars")
+    ];
 
   allGrammars = builtins.attrValues builtGrammars;