tree-sitter update script: Gitlab support

pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix

@@ -4,7 +4,7 @@ self: super:
 
 let
   generatedGrammars = callPackage ./generated.nix {
-    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/grammar.nix { };
+    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/build-grammar.nix { };
   };
 
   generatedDerivations = lib.filterAttrs (_: lib.isDerivation) generatedGrammars;

pkgs/development/tools/parsing/tree-sitter/README.md

@@ -0,0 +1,41 @@
+# tree-sitter libraries, binaries & grammars
+
+This packages tree sitter and its grammars.
+
+The grammar descriptions can be found in [./grammars.toml]().
+
+## Updating tree-sitter
+
+1) change all hashes at the beginning of [./default.nix]().
+2) Update the grammars (see below)
+
+## Updating all grammars
+
+First you need a github Personal Access Token, otherwise it runs into rate limits.
+Go to https://github.com/settings/tokens and generate a classic token, copy the secret.
+
+You generate the update script and run it:
+
+```bash
+$ nix-build -A tree-sitter.updater.update-all-grammars
+$ env GITHUB_TOKEN=<secret token> ./result
+```
+
+This will prefetch all repos mentioned in [./grammars.toml]() and put their new hashes
+into the [./grammars]() directory.
+
+If a new repository was added to the `github.com/tree-sitter` organization,
+the update process will throw an error and you need to add the new repo to
+either `knownTreeSitterOrgGrammarRepos` (if it’s a grammar) or to
+`ignoredTreeSitterOrgRepos`.
+This is to make sure we always package every official grammar.
+
+## Adding a third-party grammar
+
+Add it to the `otherGrammars` section in [./grammars.toml]().
+The grammar name has to be unique among all grammars (upstream and third party).
+
+## Deleting a grammar
+
+In case a grammar needs to be removed, please remove the generated outputs
+in the [./grammar]() directory manually.

pkgs/development/tools/parsing/tree-sitter/default.nix

@@ -23,13 +23,8 @@
 , extraGrammars ? { }
 }:
 
-# TODO: move to carnix or https://github.com/kolloch/crate2nix
 let
-  # to update:
-  # 1) change all these hashes
-  # 2) nix-build -A tree-sitter.updater.update-all-grammars
-  # 3) Set GITHUB_TOKEN env variable to avoid api rate limit (Use a Personal Access Token from https://github.com/settings/tokens It does not need any permissions)
-  # 4) run the ./result script that is output by that (it updates ./grammars)
+  # to update: see ./README.md
   version = "0.20.7";
   sha256 = "sha256-5ILiN5EfJ7WpeYBiXynfcLucdp8zmxVOj4gLkaFQYts=";
   cargoSha256 = "sha256-V4frCaU5QzTx3ujdaplw7vNkosbzyXHQvE+T7ntVOtU=";
@@ -55,7 +50,7 @@ let
   builtGrammars =
     let
       change = name: grammar:
-        callPackage ./grammar.nix { } {
+        callPackage ./build-grammar.nix { } {
           language = if grammar ? language then grammar.language else name;
           inherit version;
           source = if grammar ? src then grammar.src else fetchGrammar grammar;