From 633a8f7c20179720caaa6a555dc3c34e37f9481b Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 18:04:31 +0100
Subject: [PATCH 01/14] tree-sitter: grammar.nix -> build-grammar.nix

---
 .../editors/vim/plugins/nvim-treesitter/overrides.nix           | 2 +-
 .../parsing/tree-sitter/{grammar.nix => build-grammar.nix}      | 0
 pkgs/development/tools/parsing/tree-sitter/default.nix          | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename pkgs/development/tools/parsing/tree-sitter/{grammar.nix => build-grammar.nix} (100%)

diff --git a/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix b/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix
index 2b8ba7616ae62..4fe658b8207be 100644
--- a/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix
+++ b/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix
@@ -4,7 +4,7 @@ self: super:
 
 let
   generatedGrammars = callPackage ./generated.nix {
-    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/grammar.nix { };
+    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/build-grammar.nix { };
   };
 
   generatedDerivations = lib.filterAttrs (_: lib.isDerivation) generatedGrammars;
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammar.nix b/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix
similarity index 100%
rename from pkgs/development/tools/parsing/tree-sitter/grammar.nix
rename to pkgs/development/tools/parsing/tree-sitter/build-grammar.nix
diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index dcc8be0640f47..ce604fa19a7e8 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -55,7 +55,7 @@ let
   builtGrammars =
     let
       change = name: grammar:
-        callPackage ./grammar.nix { } {
+        callPackage ./build-grammar.nix { } {
           language = if grammar ? language then grammar.language else name;
           inherit version;
           source = if grammar ? src then grammar.src else fetchGrammar grammar;

From 139377f7db01293e457b3f8f789d592527fe8dd7 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 18:07:17 +0100
Subject: [PATCH 02/14] tree-sitter/grammars: add README.md with note of
 autogeneration

---
 pkgs/development/tools/parsing/tree-sitter/grammars/README.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/README.md

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/README.md b/pkgs/development/tools/parsing/tree-sitter/grammars/README.md
new file mode 100644
index 0000000000000..c5cb0521a1efa
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/README.md
@@ -0,0 +1,4 @@
+# Generated tree-sitter grammars
+
+These grammars have been autogenerated by the update script.
+Please see [../default.nix]() for how to run it.

From 8bc700bf496daeae33188468ad36a7545dd3dd67 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 18:51:49 +0100
Subject: [PATCH 03/14] tree-sitter/update: move grammar definitions to
 grammars.toml

---
 .../tools/parsing/tree-sitter/grammars.toml   | 270 ++++++++++++++
 .../tools/parsing/tree-sitter/update.nix      | 352 +-----------------
 2 files changed, 274 insertions(+), 348 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars.toml

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.toml b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
new file mode 100644
index 0000000000000..343db97a0cc82
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
@@ -0,0 +1,270 @@
+# Grammar list:
+# https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
+
+# Grammars we want to fetch from the tree-sitter github orga
+knownTreeSitterOrgGrammarRepos = [
+  "tree-sitter-javascript",
+  "tree-sitter-c",
+  "tree-sitter-json",
+  "tree-sitter-cpp",
+  "tree-sitter-ruby",
+  "tree-sitter-go",
+  "tree-sitter-c-sharp",
+  "tree-sitter-python",
+  "tree-sitter-typescript",
+  "tree-sitter-rust",
+  "tree-sitter-bash",
+  "tree-sitter-php",
+  "tree-sitter-java",
+  "tree-sitter-scala",
+  "tree-sitter-ocaml",
+  "tree-sitter-julia",
+  "tree-sitter-agda",
+  "tree-sitter-fluent",
+  "tree-sitter-html",
+  "tree-sitter-haskell",
+  "tree-sitter-regex",
+  "tree-sitter-css",
+  "tree-sitter-verilog",
+  "tree-sitter-jsdoc",
+  "tree-sitter-ql",
+  "tree-sitter-ql-dbscheme",
+  "tree-sitter-embedded-template",
+  "tree-sitter-tsq",
+  "tree-sitter-toml",
+]
+# repos of the tree-sitter github orga we want to ignore (not grammars)
+ignoredTreeSitterOrgRepos = [
+  "tree-sitter",
+  "tree-sitter-cli",
+  # this is the haskell language bindings, tree-sitter-haskell is the grammar
+  "haskell-tree-sitter",
+  # this is the ruby language bindings, tree-sitter-ruby is the grammar
+  "ruby-tree-sitter.old",
+  # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
+  "rust-tree-sitter",
+  # this is the nodejs language bindings, tree-sitter-javascript is the grammar
+  "node-tree-sitter",
+  # this is the python language bindings, tree-sitter-python is the grammar
+  "py-tree-sitter",
+  # afl fuzzing for tree sitter
+  "afl-tree-sitter",
+  # archived
+  "highlight-schema",
+  # website
+  "tree-sitter.github.io",
+  # not maintained
+  "tree-sitter-razor",
+  # rust library for constructing arbitrary graph structures from source code
+  "tree-sitter-graph",
+  # abandoned
+  "tree-sitter-swift",
+]
+
+# Additional grammars that are not in the official github orga.
+# If you need a grammar that already exists in the official orga,
+# make sure to give it a different name.
+[otherGrammars.tree-sitter-beancount]
+orga = "polarmutex"
+repo = "tree-sitter-beancount"
+[otherGrammars.tree-sitter-clojure]
+orga = "sogaiu"
+repo = "tree-sitter-clojure"
+[otherGrammars.tree-sitter-comment]
+orga = "stsewd"
+repo = "tree-sitter-comment"
+[otherGrammars.tree-sitter-dart]
+orga = "usernobody14"
+repo = "tree-sitter-dart"
+[otherGrammars.tree-sitter-elisp]
+orga = "wilfred"
+repo = "tree-sitter-elisp"
+[otherGrammars.tree-sitter-nix]
+orga = "cstrahan"
+repo = "tree-sitter-nix"
+[otherGrammars.tree-sitter-latex]
+orga = "latex-lsp"
+repo = "tree-sitter-latex"
+[otherGrammars.tree-sitter-lua]
+orga = "MunifTanjim"
+repo = "tree-sitter-lua"
+[otherGrammars.tree-sitter-fennel]
+orga = "travonted"
+repo = "tree-sitter-fennel"
+[otherGrammars.tree-sitter-make]
+orga = "alemuller"
+repo = "tree-sitter-make"
+[otherGrammars.tree-sitter-markdown]
+orga = "MDeiml"
+repo = "tree-sitter-markdown"
+[otherGrammars.tree-sitter-rego]
+orga = "FallenAngel97"
+repo = "tree-sitter-rego"
+[otherGrammars.tree-sitter-rst]
+orga = "stsewd"
+repo = "tree-sitter-rst"
+[otherGrammars.tree-sitter-svelte]
+orga = "Himujjal"
+repo = "tree-sitter-svelte"
+[otherGrammars.tree-sitter-sql]
+orga = "m-novikov"
+repo = "tree-sitter-sql"
+[otherGrammars.tree-sitter-vim]
+orga = "vigoux"
+repo = "tree-sitter-viml"
+[otherGrammars.tree-sitter-yaml]
+orga = "ikatyang"
+repo = "tree-sitter-yaml"
+[otherGrammars.tree-sitter-zig]
+orga = "maxxnino"
+repo = "tree-sitter-zig"
+[otherGrammars.tree-sitter-fish]
+orga = "ram02z"
+repo = "tree-sitter-fish"
+[otherGrammars.tree-sitter-dot]
+orga = "rydesun"
+repo = "tree-sitter-dot"
+[otherGrammars.tree-sitter-norg]
+orga = "nvim-neorg"
+repo = "tree-sitter-norg"
+[otherGrammars.tree-sitter-commonlisp]
+orga = "thehamsta"
+repo = "tree-sitter-commonlisp"
+[otherGrammars.tree-sitter-cuda]
+orga = "thehamsta"
+repo = "tree-sitter-cuda"
+[otherGrammars.tree-sitter-glsl]
+orga = "thehamsta"
+repo = "tree-sitter-glsl"
+[otherGrammars.tree-sitter-dockerfile]
+orga = "camdencheek"
+repo = "tree-sitter-dockerfile"
+[otherGrammars.tree-sitter-ledger]
+orga = "cbarrete"
+repo = "tree-sitter-ledger"
+[otherGrammars.tree-sitter-gomod]
+orga = "camdencheek"
+repo = "tree-sitter-go-mod"
+[otherGrammars.tree-sitter-gowork]
+orga = "omertuc"
+repo = "tree-sitter-go-work"
+[otherGrammars.tree-sitter-graphql]
+orga = "bkegley"
+repo = "tree-sitter-graphql"
+[otherGrammars.tree-sitter-pgn]
+orga = "rolandwalker"
+repo = "tree-sitter-pgn"
+[otherGrammars.tree-sitter-perl]
+orga = "ganezdragon"
+repo = "tree-sitter-perl"
+[otherGrammars.tree-sitter-kotlin]
+orga = "fwcd"
+repo = "tree-sitter-kotlin"
+[otherGrammars.tree-sitter-scss]
+orga = "serenadeai"
+repo = "tree-sitter-scss"
+[otherGrammars.tree-sitter-erlang]
+orga = "abstractmachineslab"
+repo = "tree-sitter-erlang"
+[otherGrammars.tree-sitter-elixir]
+orga = "elixir-lang"
+repo = "tree-sitter-elixir"
+[otherGrammars.tree-sitter-surface]
+orga = "connorlay"
+repo = "tree-sitter-surface"
+[otherGrammars.tree-sitter-heex]
+orga = "connorlay"
+repo = "tree-sitter-heex"
+[otherGrammars.tree-sitter-supercollider]
+orga = "madskjeldgaard"
+repo = "tree-sitter-supercollider"
+[otherGrammars.tree-sitter-tlaplus]
+orga = "tlaplus-community"
+repo = "tree-sitter-tlaplus"
+[otherGrammars.tree-sitter-glimmer]
+orga = "alexlafroscia"
+repo = "tree-sitter-glimmer"
+[otherGrammars.tree-sitter-pug]
+orga = "zealot128"
+repo = "tree-sitter-pug"
+[otherGrammars.tree-sitter-vue]
+orga = "ikatyang"
+repo = "tree-sitter-vue"
+[otherGrammars.tree-sitter-elm]
+orga = "elm-tooling"
+repo = "tree-sitter-elm"
+[otherGrammars.tree-sitter-yang]
+orga = "hubro"
+repo = "tree-sitter-yang"
+[otherGrammars.tree-sitter-query]
+orga = "nvim-treesitter"
+repo = "tree-sitter-query"
+[otherGrammars.tree-sitter-sparql]
+orga = "bonabeavis"
+repo = "tree-sitter-sparql"
+[otherGrammars.tree-sitter-gdscript]
+orga = "prestonknopp"
+repo = "tree-sitter-gdscript"
+[otherGrammars.tree-sitter-godot-resource]
+orga = "prestonknopp"
+repo = "tree-sitter-godot-resource"
+[otherGrammars.tree-sitter-turtle]
+orga = "bonabeavis"
+repo = "tree-sitter-turtle"
+[otherGrammars.tree-sitter-devicetree]
+orga = "joelspadin"
+repo = "tree-sitter-devicetree"
+[otherGrammars.tree-sitter-r]
+orga = "r-lib"
+repo = "tree-sitter-r"
+[otherGrammars.tree-sitter-bibtex]
+orga = "latex-lsp"
+repo = "tree-sitter-bibtex"
+[otherGrammars.tree-sitter-fortran]
+orga = "stadelmanma"
+repo = "tree-sitter-fortran"
+[otherGrammars.tree-sitter-cmake]
+orga = "uyha"
+repo = "tree-sitter-cmake"
+[otherGrammars.tree-sitter-janet-simple]
+orga = "sogaiu"
+repo = "tree-sitter-janet-simple"
+[otherGrammars.tree-sitter-json5]
+orga = "joakker"
+repo = "tree-sitter-json5"
+[otherGrammars.tree-sitter-pioasm]
+orga = "leo60228"
+repo = "tree-sitter-pioasm"
+[otherGrammars.tree-sitter-hjson]
+orga = "winston0410"
+repo = "tree-sitter-hjson"
+[otherGrammars.tree-sitter-llvm]
+orga = "benwilliamgraham"
+repo = "tree-sitter-llvm"
+[otherGrammars.tree-sitter-http]
+orga = "ntbbloodbath"
+repo = "tree-sitter-http"
+[otherGrammars.tree-sitter-prisma]
+orga = "victorhqc"
+repo = "tree-sitter-prisma"
+[otherGrammars.tree-sitter-org-nvim]
+orga = "milisims"
+repo = "tree-sitter-org"
+[otherGrammars.tree-sitter-hcl]
+orga = "MichaHoffmann"
+repo = "tree-sitter-hcl"
+[otherGrammars.tree-sitter-scheme]
+orga = "6cdh"
+repo = "tree-sitter-scheme"
+[otherGrammars.tree-sitter-tiger]
+orga = "ambroisie"
+repo = "tree-sitter-tiger"
+[otherGrammars.tree-sitter-nickel]
+orga = "nickel-lang"
+repo = "tree-sitter-nickel"
+[otherGrammars.tree-sitter-smithy]
+orga = "indoorvivants"
+repo = "tree-sitter-smithy"
+[otherGrammars.tree-sitter-jsonnet]
+orga = "sourcegraph"
+repo = "tree-sitter-jsonnet"
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index ad62530ee31d7..ff4f7feeb2daf 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -9,350 +9,10 @@
 , xe
 }:
 
-# Grammar list:
-# https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
 
 let
-  # Grammars we want to fetch from the tree-sitter github orga
-  knownTreeSitterOrgGrammarRepos = [
-    "tree-sitter-javascript"
-    "tree-sitter-c"
-    "tree-sitter-json"
-    "tree-sitter-cpp"
-    "tree-sitter-ruby"
-    "tree-sitter-go"
-    "tree-sitter-c-sharp"
-    "tree-sitter-python"
-    "tree-sitter-typescript"
-    "tree-sitter-rust"
-    "tree-sitter-bash"
-    "tree-sitter-php"
-    "tree-sitter-java"
-    "tree-sitter-scala"
-    "tree-sitter-ocaml"
-    "tree-sitter-julia"
-    "tree-sitter-agda"
-    "tree-sitter-fluent"
-    "tree-sitter-html"
-    "tree-sitter-haskell"
-    "tree-sitter-regex"
-    "tree-sitter-css"
-    "tree-sitter-verilog"
-    "tree-sitter-jsdoc"
-    "tree-sitter-ql"
-    "tree-sitter-ql-dbscheme"
-    "tree-sitter-embedded-template"
-    "tree-sitter-tsq"
-    "tree-sitter-toml"
-  ];
-  knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
+  grammarsToml = builtins.fromTOML (builtins.readFile ./grammars.toml);
 
-  # repos of the tree-sitter github orga we want to ignore (not grammars)
-  ignoredTreeSitterOrgRepos = [
-    "tree-sitter"
-    "tree-sitter-cli"
-    # this is the haskell language bindings, tree-sitter-haskell is the grammar
-    "haskell-tree-sitter"
-    # this is the ruby language bindings, tree-sitter-ruby is the grammar
-    "ruby-tree-sitter.old"
-    # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
-    "rust-tree-sitter"
-    # this is the nodejs language bindings, tree-sitter-javascript is the grammar
-    "node-tree-sitter"
-    # this is the python language bindings, tree-sitter-python is the grammar
-    "py-tree-sitter"
-    # afl fuzzing for tree sitter
-    "afl-tree-sitter"
-    # archived
-    "highlight-schema"
-    # website
-    "tree-sitter.github.io"
-    # not maintained
-    "tree-sitter-razor"
-    # rust library for constructing arbitrary graph structures from source code
-    "tree-sitter-graph"
-    # abandoned
-    "tree-sitter-swift"
-  ];
-  ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
-
-  # Additional grammars that are not in the official github orga.
-  # If you need a grammar that already exists in the official orga,
-  # make sure to give it a different name.
-  otherGrammars = {
-    "tree-sitter-beancount" = {
-      orga = "polarmutex";
-      repo = "tree-sitter-beancount";
-    };
-    "tree-sitter-clojure" = {
-      orga = "sogaiu";
-      repo = "tree-sitter-clojure";
-    };
-    "tree-sitter-comment" = {
-      orga = "stsewd";
-      repo = "tree-sitter-comment";
-    };
-    "tree-sitter-dart" = {
-      orga = "usernobody14";
-      repo = "tree-sitter-dart";
-    };
-    "tree-sitter-elisp" = {
-      orga = "wilfred";
-      repo = "tree-sitter-elisp";
-    };
-    "tree-sitter-nix" = {
-      orga = "cstrahan";
-      repo = "tree-sitter-nix";
-    };
-    "tree-sitter-latex" = {
-      orga = "latex-lsp";
-      repo = "tree-sitter-latex";
-    };
-    "tree-sitter-lua" = {
-      orga = "MunifTanjim";
-      repo = "tree-sitter-lua";
-    };
-    "tree-sitter-fennel" = {
-      orga = "travonted";
-      repo = "tree-sitter-fennel";
-    };
-    "tree-sitter-make" = {
-      orga = "alemuller";
-      repo = "tree-sitter-make";
-    };
-    "tree-sitter-markdown" = {
-      orga = "MDeiml";
-      repo = "tree-sitter-markdown";
-    };
-    "tree-sitter-rego" = {
-      orga = "FallenAngel97";
-      repo = "tree-sitter-rego";
-    };
-    "tree-sitter-rst" = {
-      orga = "stsewd";
-      repo = "tree-sitter-rst";
-    };
-    "tree-sitter-svelte" = {
-      orga = "Himujjal";
-      repo = "tree-sitter-svelte";
-    };
-    "tree-sitter-sql" = {
-      orga = "m-novikov";
-      repo = "tree-sitter-sql";
-    };
-    "tree-sitter-vim" = {
-      orga = "vigoux";
-      repo = "tree-sitter-viml";
-    };
-    "tree-sitter-yaml" = {
-      orga = "ikatyang";
-      repo = "tree-sitter-yaml";
-    };
-    "tree-sitter-zig" = {
-      orga = "maxxnino";
-      repo = "tree-sitter-zig";
-    };
-    "tree-sitter-fish" = {
-      orga = "ram02z";
-      repo = "tree-sitter-fish";
-    };
-    "tree-sitter-dot" = {
-      orga = "rydesun";
-      repo = "tree-sitter-dot";
-    };
-    "tree-sitter-norg" = {
-      orga = "nvim-neorg";
-      repo = "tree-sitter-norg";
-    };
-    "tree-sitter-commonlisp" = {
-      orga = "thehamsta";
-      repo = "tree-sitter-commonlisp";
-    };
-    "tree-sitter-cuda" = {
-      orga = "thehamsta";
-      repo = "tree-sitter-cuda";
-    };
-    "tree-sitter-glsl" = {
-      orga = "thehamsta";
-      repo = "tree-sitter-glsl";
-    };
-    "tree-sitter-dockerfile" = {
-      orga = "camdencheek";
-      repo = "tree-sitter-dockerfile";
-    };
-    "tree-sitter-ledger" = {
-      orga = "cbarrete";
-      repo = "tree-sitter-ledger";
-    };
-    "tree-sitter-gomod" = {
-      orga = "camdencheek";
-      repo = "tree-sitter-go-mod";
-    };
-    "tree-sitter-gowork" = {
-      orga = "omertuc";
-      repo = "tree-sitter-go-work";
-    };
-    "tree-sitter-graphql" = {
-      orga = "bkegley";
-      repo = "tree-sitter-graphql";
-    };
-    "tree-sitter-pgn" = {
-      orga = "rolandwalker";
-      repo = "tree-sitter-pgn";
-    };
-    "tree-sitter-perl" = {
-      orga = "ganezdragon";
-      repo = "tree-sitter-perl";
-    };
-    "tree-sitter-kotlin" = {
-      orga = "fwcd";
-      repo = "tree-sitter-kotlin";
-    };
-    "tree-sitter-scss" = {
-      orga = "serenadeai";
-      repo = "tree-sitter-scss";
-    };
-    "tree-sitter-erlang" = {
-      orga = "abstractmachineslab";
-      repo = "tree-sitter-erlang";
-    };
-    "tree-sitter-elixir" = {
-      orga = "elixir-lang";
-      repo = "tree-sitter-elixir";
-    };
-    "tree-sitter-surface" = {
-      orga = "connorlay";
-      repo = "tree-sitter-surface";
-    };
-    "tree-sitter-heex" = {
-      orga = "connorlay";
-      repo = "tree-sitter-heex";
-    };
-    "tree-sitter-supercollider" = {
-      orga = "madskjeldgaard";
-      repo = "tree-sitter-supercollider";
-    };
-    "tree-sitter-tlaplus" = {
-      orga = "tlaplus-community";
-      repo = "tree-sitter-tlaplus";
-    };
-    "tree-sitter-glimmer" = {
-      orga = "alexlafroscia";
-      repo = "tree-sitter-glimmer";
-    };
-    "tree-sitter-pug" = {
-      orga = "zealot128";
-      repo = "tree-sitter-pug";
-    };
-    "tree-sitter-vue" = {
-      orga = "ikatyang";
-      repo = "tree-sitter-vue";
-    };
-    "tree-sitter-elm" = {
-      orga = "elm-tooling";
-      repo = "tree-sitter-elm";
-    };
-    "tree-sitter-yang" = {
-      orga = "hubro";
-      repo = "tree-sitter-yang";
-    };
-    "tree-sitter-query" = {
-      orga = "nvim-treesitter";
-      repo = "tree-sitter-query";
-    };
-    "tree-sitter-sparql" = {
-      orga = "bonabeavis";
-      repo = "tree-sitter-sparql";
-    };
-    "tree-sitter-gdscript" = {
-      orga = "prestonknopp";
-      repo = "tree-sitter-gdscript";
-    };
-    "tree-sitter-godot-resource" = {
-      orga = "prestonknopp";
-      repo = "tree-sitter-godot-resource";
-    };
-    "tree-sitter-turtle" = {
-      orga = "bonabeavis";
-      repo = "tree-sitter-turtle";
-    };
-    "tree-sitter-devicetree" = {
-      orga = "joelspadin";
-      repo = "tree-sitter-devicetree";
-    };
-    "tree-sitter-r" = {
-      orga = "r-lib";
-      repo = "tree-sitter-r";
-    };
-    "tree-sitter-bibtex" = {
-      orga = "latex-lsp";
-      repo = "tree-sitter-bibtex";
-    };
-    "tree-sitter-fortran" = {
-      orga = "stadelmanma";
-      repo = "tree-sitter-fortran";
-    };
-    "tree-sitter-cmake" = {
-      orga = "uyha";
-      repo = "tree-sitter-cmake";
-    };
-    "tree-sitter-janet-simple" = {
-      orga = "sogaiu";
-      repo = "tree-sitter-janet-simple";
-    };
-    "tree-sitter-json5" = {
-      orga = "joakker";
-      repo = "tree-sitter-json5";
-    };
-    "tree-sitter-pioasm" = {
-      orga = "leo60228";
-      repo = "tree-sitter-pioasm";
-    };
-    "tree-sitter-hjson" = {
-      orga = "winston0410";
-      repo = "tree-sitter-hjson";
-    };
-    "tree-sitter-llvm" = {
-      orga = "benwilliamgraham";
-      repo = "tree-sitter-llvm";
-    };
-    "tree-sitter-http" = {
-      orga = "ntbbloodbath";
-      repo = "tree-sitter-http";
-    };
-    "tree-sitter-prisma" = {
-      orga = "victorhqc";
-      repo = "tree-sitter-prisma";
-    };
-    "tree-sitter-org-nvim" = {
-      orga = "milisims";
-      repo = "tree-sitter-org";
-    };
-    "tree-sitter-hcl" = {
-      orga = "MichaHoffmann";
-      repo = "tree-sitter-hcl";
-    };
-    "tree-sitter-scheme" = {
-      orga = "6cdh";
-      repo = "tree-sitter-scheme";
-    };
-    "tree-sitter-tiger" = {
-      orga = "ambroisie";
-      repo = "tree-sitter-tiger";
-    };
-    "tree-sitter-nickel" = {
-      orga = "nickel-lang";
-      repo = "tree-sitter-nickel";
-    };
-    "tree-sitter-smithy" = {
-      orga = "indoorvivants";
-      repo = "tree-sitter-smithy";
-    };
-    "tree-sitter-jsonnet" = {
-      orga = "sourcegraph";
-      repo = "tree-sitter-jsonnet";
-    };
-  };
 
   allGrammars =
     let
@@ -366,10 +26,10 @@ let
                 inherit repo;
               };
             })
-          knownTreeSitterOrgGrammarRepos);
+          grammarsToml.knownTreeSitterOrgGrammarRepos);
 
     in
-    mergeAttrsUnique otherGrammars treeSitterOrgaGrammars;
+    mergeAttrsUnique grammarsToml.otherGrammars treeSitterOrgaGrammars;
 
   # TODO: move to lib
   mergeAttrsUnique = left: right:
@@ -394,7 +54,7 @@ let
         nix-prefetch-git = "${nix-prefetch-git}/bin/nix-prefetch-git";
         printf = "${coreutils}/bin/printf";
       };
-      inherit
+      inherit (grammarsToml)
         knownTreeSitterOrgGrammarRepos
         ignoredTreeSitterOrgRepos
         ;
@@ -410,10 +70,6 @@ let
       ${script} "$@"
   '';
 
-  foreachSh = attrs: f:
-    lib.concatMapStringsSep "\n" f
-      (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
-
   jsonNewlines = lib.concatMapStringsSep "\n" (lib.generators.toJSON {});
 
   # Run the given script for each of the attr list.

From dbb3fc9122174e78e0ec867171220af0fc00285d Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 18:53:05 +0100
Subject: [PATCH 04/14] tree-sitter: add README.md

---
 .../tools/parsing/tree-sitter/README.md       | 41 +++++++++++++++++++
 .../tools/parsing/tree-sitter/default.nix     |  7 +---
 2 files changed, 42 insertions(+), 6 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/README.md

diff --git a/pkgs/development/tools/parsing/tree-sitter/README.md b/pkgs/development/tools/parsing/tree-sitter/README.md
new file mode 100644
index 0000000000000..5d4e5a2272a7d
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/README.md
@@ -0,0 +1,41 @@
+# tree-sitter libraries, binaries & grammars
+
+This packages tree sitter and its grammars.
+
+The grammar descriptions can be found in [./grammars.toml]().
+
+## Updating tree-sitter
+
+1) change all hashes at the beginning of [./default.nix]().
+2) Update the grammars (see below)
+
+## Updating all grammars
+
+First you need a github Personal Access Token, otherwise it runs into rate limits.
+Go to https://github.com/settings/tokens and generate a classic token, copy the secret.
+
+You generate the update script and run it:
+
+```bash
+$ nix-build -A tree-sitter.updater.update-all-grammars
+$ env GITHUB_TOKEN=<secret token> ./result
+```
+
+This will prefetch all repos mentioned in [./grammars.toml]() and put their new hashes
+into the [./grammars]() directory.
+
+If a new repository was added to the `github.com/tree-sitter` organization,
+the update process will throw an error and you need to add the new repo to
+either `knownTreeSitterOrgGrammarRepos` (if it’s a grammar) or to
+`ignoredTreeSitterOrgRepos`.
+This is to make sure we always package every official grammar.
+
+## Adding a third-party grammar
+
+Add it to the `otherGrammars` section in [./grammars.toml]().
+The grammar name has to be unique among all grammars (upstream and third party).
+
+## Deleting a grammar
+
+In case a grammar needs to be removed, please remove the generated outputs
+in the [./grammar]() directory manually.
diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index ce604fa19a7e8..3c59e03db9e9b 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -23,13 +23,8 @@
 , extraGrammars ? { }
 }:
 
-# TODO: move to carnix or https://github.com/kolloch/crate2nix
 let
-  # to update:
-  # 1) change all these hashes
-  # 2) nix-build -A tree-sitter.updater.update-all-grammars
-  # 3) Set GITHUB_TOKEN env variable to avoid api rate limit (Use a Personal Access Token from https://github.com/settings/tokens It does not need any permissions)
-  # 4) run the ./result script that is output by that (it updates ./grammars)
+  # to update: see ./README.md
   version = "0.20.7";
   sha256 = "sha256-5ILiN5EfJ7WpeYBiXynfcLucdp8zmxVOj4gLkaFQYts=";
   cargoSha256 = "sha256-V4frCaU5QzTx3ujdaplw7vNkosbzyXHQvE+T7ntVOtU=";

From 93473a51b014b9adbecd133438c3f658763671ba Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 19:25:16 +0100
Subject: [PATCH 05/14] tree-sitter/update: move list construction to
 allGrammars & docs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The allGrammars had to be patched at both use-sites, so let’s pull
that into the original definition.

Adds a few more docstrings.
---
 .../tools/parsing/tree-sitter/update.nix      | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index ff4f7feeb2daf..a01ad36b34ae2 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -16,6 +16,7 @@ let
 
   allGrammars =
     let
+      # All grammars in the tree sitter orga we know of
       treeSitterOrgaGrammars =
         lib.listToAttrs (map
           (repo:
@@ -28,8 +29,18 @@ let
             })
           grammarsToml.knownTreeSitterOrgGrammarRepos);
 
+      merged =
+        mergeAttrsUnique
+          grammarsToml.otherGrammars
+          treeSitterOrgaGrammars;
     in
-    mergeAttrsUnique grammarsToml.otherGrammars treeSitterOrgaGrammars;
+      # a list of {nixRepoAttrName, orga, repo}
+      lib.mapAttrsToList
+        (nixRepoAttrName: attrs: attrs // {
+          inherit nixRepoAttrName;
+        })
+        merged;
+
 
   # TODO: move to lib
   mergeAttrsUnique = left: right:
@@ -45,8 +56,6 @@ let
 
 
 
-  jsonFile = name: val: (formats.json { }).generate name val;
-
   # implementation of the updater
   updateImpl = passArgs "updateImpl-with-args" {
       binaries = {
@@ -70,7 +79,11 @@ let
       ${script} "$@"
   '';
 
+  # a list of nix values as a newline-separated json string,
+  # one entry per line
   jsonNewlines = lib.concatMapStringsSep "\n" (lib.generators.toJSON {});
+  # a pretty-printed value as json file
+  jsonFile = name: val: (formats.json { }).generate name val;
 
   # Run the given script for each of the attr list.
   # The attrs are passed to the script as a json value.
@@ -83,9 +96,14 @@ let
   # This will depend on your local environment, but that is intentional.
   outputDir = "${toString ./.}/grammars";
 
+  # final script
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
-   ${updateImpl} fetch-and-check-tree-sitter-repos '{}'
+
+    # first make sure we know about all upsteam repos
+    ${updateImpl} fetch-and-check-tree-sitter-repos '{}'
+
+    # Then write one json file for each prefetched repo, in parallel
     echo "writing files to ${outputDir}" 1>&2
     mkdir -p "${outputDir}"
     ${forEachParallel
@@ -93,23 +111,17 @@ let
         (writeShellScript "fetch-repo" ''
             ${updateImpl} fetch-repo "$1"
         '')
-        (lib.mapAttrsToList
-          (nixRepoAttrName: attrs: attrs // {
-            inherit
-              nixRepoAttrName
-              outputDir;
-          })
+        (map
+          (grammar: grammar // { inherit outputDir; })
           allGrammars)
     }
+
+    # finally, write a default.nix that calls all grammars
     ${updateImpl} print-all-grammars-nix-file "$(< ${
         jsonFile "all-grammars.json" {
-          allGrammars =
-            (lib.mapAttrsToList
-              (nixRepoAttrName: attrs: attrs // {
-                inherit nixRepoAttrName;
-              })
-              allGrammars);
-          inherit outputDir;
+          inherit
+            allGrammars
+            outputDir;
         }
     })"
   '';

From 87a1b06eb83ea0bb7c5d1e9c0d9d257205d3569e Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 20:04:31 +0100
Subject: [PATCH 06/14] tree-sitter/update: only pass necessary data to final
 print command

`repo` and `orga` are not required to print the `default.nix`. This
will be relevant because `gitlab` is going to be added, which does not
have these two fields.
---
 .../tools/parsing/tree-sitter/update.nix         | 10 ++++++----
 .../tools/parsing/tree-sitter/update_impl.py     | 16 ++++------------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index a01ad36b34ae2..4d1d12e6edb97 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -118,10 +118,12 @@ let
 
     # finally, write a default.nix that calls all grammars
     ${updateImpl} print-all-grammars-nix-file "$(< ${
-        jsonFile "all-grammars.json" {
-          inherit
-            allGrammars
-            outputDir;
+        jsonFile "repo-attr-names.json" {
+          repoAttrNames =
+            map
+              ({nixRepoAttrName, ...}: nixRepoAttrName)
+              allGrammars;
+          inherit outputDir;
         }
     })"
   '';
diff --git a/pkgs/development/tools/parsing/tree-sitter/update_impl.py b/pkgs/development/tools/parsing/tree-sitter/update_impl.py
index db470617ed9c2..b4bd7b7bdf125 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update_impl.py
+++ b/pkgs/development/tools/parsing/tree-sitter/update_impl.py
@@ -3,7 +3,7 @@
 import subprocess as sub
 import os
 import sys
-from typing import Iterator, Any, Literal, TypedDict
+from typing import Iterator, Any, Literal
 from tempfile import NamedTemporaryFile
 
 debug: bool = True if os.environ.get("DEBUG", False) else False
@@ -166,26 +166,18 @@ def checkTreeSitterRepos(latest_github_repos: set[str]) -> None:
         sys.exit(f"These repositories are neither known nor ignored:\n{unknown}")
 
 
-Grammar = TypedDict(
-    "Grammar",
-    {
-        "nixRepoAttrName": str,
-        "orga": str,
-        "repo": str
-    }
-)
+NixRepoAttrName = str
 
 
 def printAllGrammarsNixFile() -> None:
     """Print a .nix file that imports all grammars."""
-    allGrammars: list[dict[str, Grammar]] = jsonArg["allGrammars"]
+    repoAttrNames: list[NixRepoAttrName] = jsonArg["repoAttrNames"]
     outputDir: Dir = jsonArg["outputDir"]
 
     def file() -> Iterator[str]:
         yield "{ lib }:"
         yield "{"
-        for grammar in allGrammars:
-            n = grammar["nixRepoAttrName"]
+        for n in repoAttrNames:
             yield f"  {n} = lib.importJSON ./{n}.json;"
         yield "}"
         yield ""

From c59357b0d22ed9d37eb767e15eefcfd1fa845df2 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 21:02:33 +0100
Subject: [PATCH 07/14] tree-sitter/update: prepare adding support for gitlab
 repos

We want to be able to download tree-sitter grammars from other
platforms. The first, easiest target is gitlab.
Gitlab uses a `projectId` instead of orga/repo though, so we need to
implement it differently (also checking the latest version of a repo
works differently).
---
 .../tools/parsing/tree-sitter/grammars.toml   |  73 ++++++++++
 .../tools/parsing/tree-sitter/update.nix      |   3 +-
 .../tools/parsing/tree-sitter/update_impl.py  | 129 +++++++++++-------
 3 files changed, 157 insertions(+), 48 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.toml b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
index 343db97a0cc82..a89d4fde7a5e4 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars.toml
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
@@ -64,207 +64,280 @@ ignoredTreeSitterOrgRepos = [
 # Additional grammars that are not in the official github orga.
 # If you need a grammar that already exists in the official orga,
 # make sure to give it a different name.
+#
+# This can be type == "github", then it needs an orga and a repo.
+#
+# Or it can be type == "gitlab", then it needs a projectId,
+# which you can find under the repository name on gitlab.
 [otherGrammars.tree-sitter-beancount]
+type = "github"
 orga = "polarmutex"
 repo = "tree-sitter-beancount"
 [otherGrammars.tree-sitter-clojure]
+type = "github"
 orga = "sogaiu"
 repo = "tree-sitter-clojure"
 [otherGrammars.tree-sitter-comment]
+type = "github"
 orga = "stsewd"
 repo = "tree-sitter-comment"
 [otherGrammars.tree-sitter-dart]
+type = "github"
 orga = "usernobody14"
 repo = "tree-sitter-dart"
 [otherGrammars.tree-sitter-elisp]
+type = "github"
 orga = "wilfred"
 repo = "tree-sitter-elisp"
 [otherGrammars.tree-sitter-nix]
+type = "github"
 orga = "cstrahan"
 repo = "tree-sitter-nix"
 [otherGrammars.tree-sitter-latex]
+type = "github"
 orga = "latex-lsp"
 repo = "tree-sitter-latex"
 [otherGrammars.tree-sitter-lua]
+type = "github"
 orga = "MunifTanjim"
 repo = "tree-sitter-lua"
 [otherGrammars.tree-sitter-fennel]
+type = "github"
 orga = "travonted"
 repo = "tree-sitter-fennel"
 [otherGrammars.tree-sitter-make]
+type = "github"
 orga = "alemuller"
 repo = "tree-sitter-make"
 [otherGrammars.tree-sitter-markdown]
+type = "github"
 orga = "MDeiml"
 repo = "tree-sitter-markdown"
 [otherGrammars.tree-sitter-rego]
+type = "github"
 orga = "FallenAngel97"
 repo = "tree-sitter-rego"
 [otherGrammars.tree-sitter-rst]
+type = "github"
 orga = "stsewd"
 repo = "tree-sitter-rst"
 [otherGrammars.tree-sitter-svelte]
+type = "github"
 orga = "Himujjal"
 repo = "tree-sitter-svelte"
 [otherGrammars.tree-sitter-sql]
+type = "github"
 orga = "m-novikov"
 repo = "tree-sitter-sql"
 [otherGrammars.tree-sitter-vim]
+type = "github"
 orga = "vigoux"
 repo = "tree-sitter-viml"
 [otherGrammars.tree-sitter-yaml]
+type = "github"
 orga = "ikatyang"
 repo = "tree-sitter-yaml"
 [otherGrammars.tree-sitter-zig]
+type = "github"
 orga = "maxxnino"
 repo = "tree-sitter-zig"
 [otherGrammars.tree-sitter-fish]
+type = "github"
 orga = "ram02z"
 repo = "tree-sitter-fish"
 [otherGrammars.tree-sitter-dot]
+type = "github"
 orga = "rydesun"
 repo = "tree-sitter-dot"
 [otherGrammars.tree-sitter-norg]
+type = "github"
 orga = "nvim-neorg"
 repo = "tree-sitter-norg"
 [otherGrammars.tree-sitter-commonlisp]
+type = "github"
 orga = "thehamsta"
 repo = "tree-sitter-commonlisp"
 [otherGrammars.tree-sitter-cuda]
+type = "github"
 orga = "thehamsta"
 repo = "tree-sitter-cuda"
 [otherGrammars.tree-sitter-glsl]
+type = "github"
 orga = "thehamsta"
 repo = "tree-sitter-glsl"
 [otherGrammars.tree-sitter-dockerfile]
+type = "github"
 orga = "camdencheek"
 repo = "tree-sitter-dockerfile"
 [otherGrammars.tree-sitter-ledger]
+type = "github"
 orga = "cbarrete"
 repo = "tree-sitter-ledger"
 [otherGrammars.tree-sitter-gomod]
+type = "github"
 orga = "camdencheek"
 repo = "tree-sitter-go-mod"
 [otherGrammars.tree-sitter-gowork]
+type = "github"
 orga = "omertuc"
 repo = "tree-sitter-go-work"
 [otherGrammars.tree-sitter-graphql]
+type = "github"
 orga = "bkegley"
 repo = "tree-sitter-graphql"
 [otherGrammars.tree-sitter-pgn]
+type = "github"
 orga = "rolandwalker"
 repo = "tree-sitter-pgn"
 [otherGrammars.tree-sitter-perl]
+type = "github"
 orga = "ganezdragon"
 repo = "tree-sitter-perl"
 [otherGrammars.tree-sitter-kotlin]
+type = "github"
 orga = "fwcd"
 repo = "tree-sitter-kotlin"
 [otherGrammars.tree-sitter-scss]
+type = "github"
 orga = "serenadeai"
 repo = "tree-sitter-scss"
 [otherGrammars.tree-sitter-erlang]
+type = "github"
 orga = "abstractmachineslab"
 repo = "tree-sitter-erlang"
 [otherGrammars.tree-sitter-elixir]
+type = "github"
 orga = "elixir-lang"
 repo = "tree-sitter-elixir"
 [otherGrammars.tree-sitter-surface]
+type = "github"
 orga = "connorlay"
 repo = "tree-sitter-surface"
 [otherGrammars.tree-sitter-heex]
+type = "github"
 orga = "connorlay"
 repo = "tree-sitter-heex"
 [otherGrammars.tree-sitter-supercollider]
+type = "github"
 orga = "madskjeldgaard"
 repo = "tree-sitter-supercollider"
 [otherGrammars.tree-sitter-tlaplus]
+type = "github"
 orga = "tlaplus-community"
 repo = "tree-sitter-tlaplus"
 [otherGrammars.tree-sitter-glimmer]
+type = "github"
 orga = "alexlafroscia"
 repo = "tree-sitter-glimmer"
 [otherGrammars.tree-sitter-pug]
+type = "github"
 orga = "zealot128"
 repo = "tree-sitter-pug"
 [otherGrammars.tree-sitter-vue]
+type = "github"
 orga = "ikatyang"
 repo = "tree-sitter-vue"
 [otherGrammars.tree-sitter-elm]
+type = "github"
 orga = "elm-tooling"
 repo = "tree-sitter-elm"
 [otherGrammars.tree-sitter-yang]
+type = "github"
 orga = "hubro"
 repo = "tree-sitter-yang"
 [otherGrammars.tree-sitter-query]
+type = "github"
 orga = "nvim-treesitter"
 repo = "tree-sitter-query"
 [otherGrammars.tree-sitter-sparql]
+type = "github"
 orga = "bonabeavis"
 repo = "tree-sitter-sparql"
 [otherGrammars.tree-sitter-gdscript]
+type = "github"
 orga = "prestonknopp"
 repo = "tree-sitter-gdscript"
 [otherGrammars.tree-sitter-godot-resource]
+type = "github"
 orga = "prestonknopp"
 repo = "tree-sitter-godot-resource"
 [otherGrammars.tree-sitter-turtle]
+type = "github"
 orga = "bonabeavis"
 repo = "tree-sitter-turtle"
 [otherGrammars.tree-sitter-devicetree]
+type = "github"
 orga = "joelspadin"
 repo = "tree-sitter-devicetree"
 [otherGrammars.tree-sitter-r]
+type = "github"
 orga = "r-lib"
 repo = "tree-sitter-r"
 [otherGrammars.tree-sitter-bibtex]
+type = "github"
 orga = "latex-lsp"
 repo = "tree-sitter-bibtex"
 [otherGrammars.tree-sitter-fortran]
+type = "github"
 orga = "stadelmanma"
 repo = "tree-sitter-fortran"
 [otherGrammars.tree-sitter-cmake]
+type = "github"
 orga = "uyha"
 repo = "tree-sitter-cmake"
 [otherGrammars.tree-sitter-janet-simple]
+type = "github"
 orga = "sogaiu"
 repo = "tree-sitter-janet-simple"
 [otherGrammars.tree-sitter-json5]
+type = "github"
 orga = "joakker"
 repo = "tree-sitter-json5"
 [otherGrammars.tree-sitter-pioasm]
+type = "github"
 orga = "leo60228"
 repo = "tree-sitter-pioasm"
 [otherGrammars.tree-sitter-hjson]
+type = "github"
 orga = "winston0410"
 repo = "tree-sitter-hjson"
 [otherGrammars.tree-sitter-llvm]
+type = "github"
 orga = "benwilliamgraham"
 repo = "tree-sitter-llvm"
 [otherGrammars.tree-sitter-http]
+type = "github"
 orga = "ntbbloodbath"
 repo = "tree-sitter-http"
 [otherGrammars.tree-sitter-prisma]
+type = "github"
 orga = "victorhqc"
 repo = "tree-sitter-prisma"
 [otherGrammars.tree-sitter-org-nvim]
+type = "github"
 orga = "milisims"
 repo = "tree-sitter-org"
 [otherGrammars.tree-sitter-hcl]
+type = "github"
 orga = "MichaHoffmann"
 repo = "tree-sitter-hcl"
 [otherGrammars.tree-sitter-scheme]
+type = "github"
 orga = "6cdh"
 repo = "tree-sitter-scheme"
 [otherGrammars.tree-sitter-tiger]
+type = "github"
 orga = "ambroisie"
 repo = "tree-sitter-tiger"
 [otherGrammars.tree-sitter-nickel]
+type = "github"
 orga = "nickel-lang"
 repo = "tree-sitter-nickel"
 [otherGrammars.tree-sitter-smithy]
+type = "github"
 orga = "indoorvivants"
 repo = "tree-sitter-smithy"
 [otherGrammars.tree-sitter-jsonnet]
+type = "github"
 orga = "sourcegraph"
 repo = "tree-sitter-jsonnet"
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 4d1d12e6edb97..4f140c9497334 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -14,6 +14,7 @@ let
   grammarsToml = builtins.fromTOML (builtins.readFile ./grammars.toml);
 
 
+  # a list of {nixRepoAttrName, type, <type-specific>}
   allGrammars =
     let
       # All grammars in the tree sitter orga we know of
@@ -23,6 +24,7 @@ let
             {
               name = repo;
               value = {
+                type = "github";
                 orga = "tree-sitter";
                 inherit repo;
               };
@@ -34,7 +36,6 @@ let
           grammarsToml.otherGrammars
           treeSitterOrgaGrammars;
     in
-      # a list of {nixRepoAttrName, orga, repo}
       lib.mapAttrsToList
         (nixRepoAttrName: attrs: attrs // {
           inherit nixRepoAttrName;
diff --git a/pkgs/development/tools/parsing/tree-sitter/update_impl.py b/pkgs/development/tools/parsing/tree-sitter/update_impl.py
index b4bd7b7bdf125..824c0b82f2f10 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update_impl.py
+++ b/pkgs/development/tools/parsing/tree-sitter/update_impl.py
@@ -3,7 +3,7 @@
 import subprocess as sub
 import os
 import sys
-from typing import Iterator, Any, Literal
+from typing import Iterator, Any, Literal, NoReturn, TypedDict, cast
 from tempfile import NamedTemporaryFile
 
 debug: bool = True if os.environ.get("DEBUG", False) else False
@@ -21,6 +21,10 @@ def log(msg: str) -> None:
     print(msg, file=sys.stderr)
 
 
+def critical(msg: str) -> NoReturn:
+    sys.exit(f"ERROR: {msg}")
+
+
 def atomically_write(file_path: str, content: bytes) -> None:
     """atomically write the content into `file_path`"""
     with NamedTemporaryFile(
@@ -58,7 +62,7 @@ def curl_result(output: bytes) -> Any | Literal["not found"]:
         case dict(res):
             message: str = res.get("message", "")
             if "rate limit" in message:
-                sys.exit("Rate limited by the Github API")
+                critical("Rate limited by the Github API")
             if "Not Found" in message:
                 return "not found"
     # if the result is another type, we can pass it on
@@ -86,50 +90,81 @@ def run_cmd(args: Args) -> bytes:
 
 Dir = str
 
+GithubRepo = TypedDict(
+    "GithubRepo", {
+        "orga": str,
+        "repo": str
+    }
+)
+
+GitlabRepo = TypedDict(
+    "GitlabRepo", {
+        "projectId": str
+    }
+)
+
+FetchRepoArg = TypedDict(
+    "FetchRepoArg", {
+        "type": str,
+        "outputDir": Dir,
+        "nixRepoAttrName": str
+    }
+
+)
+
 
 def fetchRepo() -> None:
     """fetch the given repo and write its nix-prefetch output to the corresponding grammar json file"""
-    match jsonArg:
-        case {
-            "orga": orga,
-            "repo": repo,
-            "outputDir": outputDir,
-            "nixRepoAttrName": nixRepoAttrName,
-        }:
-            token: str | None = os.environ.get("GITHUB_TOKEN", None)
-            out = run_cmd(
-                curl_github_args(
-                    token,
-                    url=f"https://api.github.com/repos/{quote(orga)}/{quote(repo)}/releases/latest"
-                )
-            )
-            release: str
-            match curl_result(out):
-                case "not found":
-                    # github sometimes returns an empty list even tough there are releases
-                    log(f"uh-oh, latest for {orga}/{repo} is not there, using HEAD")
-                    release = "HEAD"
-                case {"tag_name": tag_name}:
-                    release = tag_name
-                case _:
-                    sys.exit(f"git result for {orga}/{repo} did not have a `tag_name` field")
-
-            log(f"Fetching latest release ({release}) of {orga}/{repo} …")
-            res = run_cmd(
-                nix_prefetch_git_args(
-                    url=f"https://github.com/{quote(orga)}/{quote(repo)}",
-                    version_rev=release
-                )
-            )
-            atomically_write(
-                file_path=os.path.join(
-                    outputDir,
-                    f"{nixRepoAttrName}.json"
-                ),
-                content=res
-            )
+    arg = cast(FetchRepoArg, jsonArg)
+    if debug:
+        log(f"Fetching repo {arg}")
+    match arg["type"]:
+        case "github":
+            repo = cast(GithubRepo, jsonArg)
+            res = fetchGithubRepo(repo)
+        case "gitlab":
+            # repo = cast(GitlabRepo, jsonArg)
+            critical("gitlab repos not yet implemented")
+        case other:
+            critical(f'''Do not yet know how to handle the repo type "{other}"''')
+    attrName = jsonArg["nixRepoAttrName"]
+    atomically_write(
+        file_path=os.path.join(
+            arg["outputDir"],
+            f"{attrName}.json"
+        ),
+        content=res
+    )
+
+
+def fetchGithubRepo(r: GithubRepo) -> bytes:
+    token: str | None = os.environ.get("GITHUB_TOKEN", None)
+    orga = r["orga"]
+    repo = r["repo"]
+    out = run_cmd(
+        curl_github_args(
+            token,
+            url=f"https://api.github.com/repos/{quote(orga)}/{quote(repo)}/releases/latest"
+        )
+    )
+    release: str
+    match curl_result(out):
+        case "not found":
+            # github sometimes returns an empty list even tough there are releases
+            log(f"uh-oh, latest for {orga}/{repo} is not there, using HEAD")
+            release = "HEAD"
+        case {"tag_name": tag_name}:
+            release = tag_name
         case _:
-            sys.exit("input json must have `orga` and `repo` keys")
+            critical(f"git result for {orga}/{repo} did not have a `tag_name` field")
+
+    log(f"Fetching latest release ({release}) of {orga}/{repo} …")
+    return run_cmd(
+        nix_prefetch_git_args(
+            url=f"https://github.com/{quote(orga)}/{quote(repo)}",
+            version_rev=release
+        )
+    )
 
 
 def fetchOrgaLatestRepos(orga: str) -> set[str]:
@@ -143,7 +178,7 @@ def fetchOrgaLatestRepos(orga: str) -> set[str]:
     )
     match curl_result(out):
         case "not found":
-            sys.exit(f"github organization {orga} not found")
+            critical(f"github organization {orga} not found")
         case list(repos):
             res: list[str] = []
             for repo in repos:
@@ -151,8 +186,8 @@ def fetchOrgaLatestRepos(orga: str) -> set[str]:
                 if name:
                     res.append(name)
             return set(res)
-        case _:
-            sys.exit("github result was not a list of repos, but {other}")
+        case other:
+            critical(f"github result was not a list of repos, but {other}")
 
 
 def checkTreeSitterRepos(latest_github_repos: set[str]) -> None:
@@ -163,7 +198,7 @@ def checkTreeSitterRepos(latest_github_repos: set[str]) -> None:
     unknown = latest_github_repos - (known | ignored)
 
     if unknown:
-        sys.exit(f"These repositories are neither known nor ignored:\n{unknown}")
+        critical(f"These repositories are neither known nor ignored:\n{unknown}")
 
 
 NixRepoAttrName = str
@@ -206,4 +241,4 @@ def fetchAndCheckTreeSitterRepos() -> None:
     case "print-all-grammars-nix-file":
         printAllGrammarsNixFile()
     case _:
-        sys.exit(f"mode {mode} unknown")
+        critical(f"mode {mode} unknown")

From 8de3f17520a1abdb255e2ceb9ebb2b80c18efc9a Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Fri, 4 Nov 2022 21:55:51 +0100
Subject: [PATCH 08/14] tree-sitter/update: implement gitlab support & add
 jsonc grammar
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This uses the gitlab API to download the jsonc grammar.

Currently the gitlab support only allows for gitlab.com, but that
could be changed quite easily.

I opted for copying a lot of the github code, since it’s too early to
factor out shared code.
---
 .../tools/parsing/tree-sitter/grammars.toml   |  3 +
 .../parsing/tree-sitter/grammars/default.nix  |  1 +
 .../grammars/tree-sitter-jsonc.json           | 11 +++
 .../tools/parsing/tree-sitter/update_impl.py  | 68 ++++++++++++++++---
 4 files changed, 75 insertions(+), 8 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsonc.json

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.toml b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
index a89d4fde7a5e4..9f9ecd308e3f6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars.toml
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
@@ -341,3 +341,6 @@ repo = "tree-sitter-smithy"
 type = "github"
 orga = "sourcegraph"
 repo = "tree-sitter-jsonnet"
+[otherGrammars.tree-sitter-jsonc]
+type = "gitlab"
+projectId = "24426815"
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
index 5b3862082d8f7..7854e69723353 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
@@ -46,6 +46,7 @@
   tree-sitter-jsdoc = lib.importJSON ./tree-sitter-jsdoc.json;
   tree-sitter-json = lib.importJSON ./tree-sitter-json.json;
   tree-sitter-json5 = lib.importJSON ./tree-sitter-json5.json;
+  tree-sitter-jsonc = lib.importJSON ./tree-sitter-jsonc.json;
   tree-sitter-jsonnet = lib.importJSON ./tree-sitter-jsonnet.json;
   tree-sitter-julia = lib.importJSON ./tree-sitter-julia.json;
   tree-sitter-kotlin = lib.importJSON ./tree-sitter-kotlin.json;
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsonc.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsonc.json
new file mode 100644
index 0000000000000..f29175254b485
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsonc.json
@@ -0,0 +1,11 @@
+{
+  "url": "https://gitlab.com/WhyNotHugo/tree-sitter-jsonc.git",
+  "rev": "02b01653c8a1c198ae7287d566efa86a135b30d5",
+  "date": "2021-03-07T20:32:20+01:00",
+  "path": "/nix/store/hqh4kxw3fp9hr3yglsxv9d2kvcvpzdfa-tree-sitter-jsonc",
+  "sha256": "0mc68i7shwmn88iv3lcqyjvrhy3b62h02k272is7chk2yiw3crw9",
+  "fetchLFS": false,
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/update_impl.py b/pkgs/development/tools/parsing/tree-sitter/update_impl.py
index 824c0b82f2f10..f23c40175aee3 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update_impl.py
+++ b/pkgs/development/tools/parsing/tree-sitter/update_impl.py
@@ -55,7 +55,17 @@ def curl_github_args(token: str | None, url: str) -> Args:
     yield url
 
 
-def curl_result(output: bytes) -> Any | Literal["not found"]:
+def curl_gitlab_args(url: str) -> Args:
+    """Query the gitlab API via curl"""
+    yield bins["curl"]
+    if not debug:
+        yield "--silent"
+    # follow redirects
+    yield "--location"
+    yield url
+
+
+def github_curl_result(output: bytes) -> Any | Literal["not found"]:
     """Parse the curl result of the github API"""
     res: Any = json.loads(output)
     match res:
@@ -69,6 +79,12 @@ def curl_result(output: bytes) -> Any | Literal["not found"]:
     return res
 
 
+def gitlab_curl_result(output: bytes) -> Any:
+    """Parse the curl result of the gitlab API"""
+    res: Any = json.loads(output)
+    return res
+
+
 def nix_prefetch_git_args(url: str, version_rev: str) -> Args:
     """Prefetch a git repository"""
     yield bins["nix-prefetch-git"]
@@ -99,6 +115,7 @@ def run_cmd(args: Args) -> bytes:
 
 GitlabRepo = TypedDict(
     "GitlabRepo", {
+        "nixRepoAttrName": str,
         "projectId": str
     }
 )
@@ -109,7 +126,6 @@ def run_cmd(args: Args) -> bytes:
         "outputDir": Dir,
         "nixRepoAttrName": str
     }
-
 )
 
 
@@ -120,11 +136,9 @@ def fetchRepo() -> None:
         log(f"Fetching repo {arg}")
     match arg["type"]:
         case "github":
-            repo = cast(GithubRepo, jsonArg)
-            res = fetchGithubRepo(repo)
+            res = fetchGithubRepo(cast(GithubRepo, jsonArg))
         case "gitlab":
-            # repo = cast(GitlabRepo, jsonArg)
-            critical("gitlab repos not yet implemented")
+            res = fetchGitlabRepo(cast(GitlabRepo, jsonArg))
         case other:
             critical(f'''Do not yet know how to handle the repo type "{other}"''')
     attrName = jsonArg["nixRepoAttrName"]
@@ -148,7 +162,7 @@ def fetchGithubRepo(r: GithubRepo) -> bytes:
         )
     )
     release: str
-    match curl_result(out):
+    match github_curl_result(out):
         case "not found":
             # github sometimes returns an empty list even tough there are releases
             log(f"uh-oh, latest for {orga}/{repo} is not there, using HEAD")
@@ -167,6 +181,44 @@ def fetchGithubRepo(r: GithubRepo) -> bytes:
     )
 
 
+def fetchGitlabRepo(r: GitlabRepo) -> bytes:
+    projectId = r["projectId"]
+    nixRepoAttrName = r["nixRepoAttrName"]
+    out = run_cmd(
+        curl_gitlab_args(
+            url=f"https://gitlab.com/api/v4/projects/{quote(projectId)}/repository/tags?order_by=version&sort=desc"
+        )
+    )
+    release: str
+    projectName = f'''"{nixRepoAttrName}" (Gitlab projectId: {projectId})'''
+    match gitlab_curl_result(out):
+        case list([]):
+            log(f"uh-oh, no release find for for {projectName}, using HEAD")
+            release = "HEAD"
+        case list([{"name": tag_name}, *_]):
+            release = tag_name
+        case _:
+            critical(f"tag list for {projectName} did not have a `name` field: {out.decode()}")
+    out = run_cmd(
+        curl_gitlab_args(
+            url=f"https://gitlab.com/api/v4/projects/{quote(projectId)}"
+        )
+    )
+    url: str
+    match gitlab_curl_result(out):
+        case {"http_url_to_repo": url}:
+            url = url
+        case _:
+            critical(f"repository result for {projectName} did not have a `http_url_to_repo` field: {out.decode()}")
+    log(f"Fetching latest release ({release}) of {projectName} …")
+    return run_cmd(
+        nix_prefetch_git_args(
+            url,
+            version_rev=release
+        )
+    )
+
+
 def fetchOrgaLatestRepos(orga: str) -> set[str]:
     """fetch the latest (100) repos from the given github organization"""
     token: str | None = os.environ.get("GITHUB_TOKEN", None)
@@ -176,7 +228,7 @@ def fetchOrgaLatestRepos(orga: str) -> set[str]:
             url=f"https://api.github.com/orgs/{quote(orga)}/repos?per_page=100"
         )
     )
-    match curl_result(out):
+    match github_curl_result(out):
         case "not found":
             critical(f"github organization {orga} not found")
         case list(repos):

From cf355011cfa2e645872462878ee62a242c5901f3 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Sat, 5 Nov 2022 19:12:05 +0100
Subject: [PATCH 09/14] tree-sitter: remove `language` name changes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The `language` attr is only used for the grammar name, changing it
makes no sense at all, and makes the output names irregular.

I read something about nvim inferring the language from the filepath
or something, I seriously hope that’s not A Thing, and if it is, we
should find a different way of solving that.

The tree-sitter integration is not specific to nvim, so it shouldn’t
have nvim hacks.
---
 .../development/tools/parsing/tree-sitter/build-grammar.nix | 6 +++++-
 pkgs/development/tools/parsing/tree-sitter/default.nix      | 6 +++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix b/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix
index a4d8d7324f993..47bbf5e9a7358 100644
--- a/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix
@@ -12,6 +12,7 @@
 , version
   # source for the language grammar
 , source
+  # subdirectory inside of source that contains the actual grammar, or `null` of none.
 , location ? null
 }:
 
@@ -20,7 +21,10 @@ stdenv.mkDerivation rec {
   pname = "${language}-grammar";
   inherit version;
 
-  src = if location == null then source else "${source}/${location}";
+  src =
+    if location == null
+    then source
+    else "${source}/${location}";
 
   buildInputs = [ tree-sitter ];
 
diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 3c59e03db9e9b..af7ea347bdc6c 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -47,11 +47,12 @@ let
     '' + (lib.concatStrings (lib.mapAttrsToList
       (name: grammar: "ln -s ${if grammar ? src then grammar.src else fetchGrammar grammar} $out/${name}\n")
       (import ./grammars { inherit lib; }))));
+
   builtGrammars =
     let
       change = name: grammar:
         callPackage ./build-grammar.nix { } {
-          language = if grammar ? language then grammar.language else name;
+          language = name;
           inherit version;
           source = if grammar ? src then grammar.src else fetchGrammar grammar;
           location = if grammar ? location then grammar.location else null;
@@ -60,11 +61,10 @@ let
       grammars = grammars' //
         { tree-sitter-ocaml = grammars'.tree-sitter-ocaml // { location = "ocaml"; }; } //
         { tree-sitter-ocaml-interface = grammars'.tree-sitter-ocaml // { location = "interface"; }; } //
-        { tree-sitter-org-nvim = grammars'.tree-sitter-org-nvim // { language = "org"; }; } //
         { tree-sitter-typescript = grammars'.tree-sitter-typescript // { location = "typescript"; }; } //
         { tree-sitter-tsx = grammars'.tree-sitter-typescript // { location = "tsx"; }; } //
         { tree-sitter-markdown = grammars'.tree-sitter-markdown // { location = "tree-sitter-markdown"; }; } //
-        { tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // { language = "markdown_inline"; location = "tree-sitter-markdown-inline"; }; };
+        { tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // { location = "tree-sitter-markdown-inline"; }; };
     in
     lib.mapAttrs change (grammars);
 

From 56597c4c36278c6f7f57535d119c3fce82514cb2 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Sat, 5 Nov 2022 21:51:42 +0100
Subject: [PATCH 10/14] tree-sitter/grammars: prepare adding more fields to
 known repos

Some of these repos define multiple grammars, so we need to be able to
specify them, in order to remove the hack in the default.nix
---
 .../tools/parsing/tree-sitter/grammars.toml   | 93 ++++++++++++-------
 .../tools/parsing/tree-sitter/update.nix      |  7 +-
 2 files changed, 66 insertions(+), 34 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.toml b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
index 9f9ecd308e3f6..488cf5b810355 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars.toml
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
@@ -1,38 +1,6 @@
 # Grammar list:
 # https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
 
-# Grammars we want to fetch from the tree-sitter github orga
-knownTreeSitterOrgGrammarRepos = [
-  "tree-sitter-javascript",
-  "tree-sitter-c",
-  "tree-sitter-json",
-  "tree-sitter-cpp",
-  "tree-sitter-ruby",
-  "tree-sitter-go",
-  "tree-sitter-c-sharp",
-  "tree-sitter-python",
-  "tree-sitter-typescript",
-  "tree-sitter-rust",
-  "tree-sitter-bash",
-  "tree-sitter-php",
-  "tree-sitter-java",
-  "tree-sitter-scala",
-  "tree-sitter-ocaml",
-  "tree-sitter-julia",
-  "tree-sitter-agda",
-  "tree-sitter-fluent",
-  "tree-sitter-html",
-  "tree-sitter-haskell",
-  "tree-sitter-regex",
-  "tree-sitter-css",
-  "tree-sitter-verilog",
-  "tree-sitter-jsdoc",
-  "tree-sitter-ql",
-  "tree-sitter-ql-dbscheme",
-  "tree-sitter-embedded-template",
-  "tree-sitter-tsq",
-  "tree-sitter-toml",
-]
 # repos of the tree-sitter github orga we want to ignore (not grammars)
 ignoredTreeSitterOrgRepos = [
   "tree-sitter",
@@ -61,6 +29,67 @@ ignoredTreeSitterOrgRepos = [
   "tree-sitter-swift",
 ]
 
+# Grammars we want to fetch from the tree-sitter github orga
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-javascript"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-c"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-json"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-cpp"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-ruby"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-go"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-c-sharp"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-python"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-typescript"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-rust"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-bash"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-php"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-java"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-scala"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-ocaml"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-julia"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-agda"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-fluent"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-html"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-haskell"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-regex"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-css"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-verilog"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-jsdoc"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-ql"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-ql-dbscheme"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-embedded-template"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-tsq"
+[[knownTreeSitterOrgGrammarRepos]]
+repo = "tree-sitter-toml"
+
+
 # Additional grammars that are not in the official github orga.
 # If you need a grammar that already exists in the official orga,
 # make sure to give it a different name.
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 4f140c9497334..0ccc9cf80bebd 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -20,7 +20,7 @@ let
       # All grammars in the tree sitter orga we know of
       treeSitterOrgaGrammars =
         lib.listToAttrs (map
-          (repo:
+          ({repo, ...}:
             {
               name = repo;
               value = {
@@ -65,9 +65,12 @@ let
         printf = "${coreutils}/bin/printf";
       };
       inherit (grammarsToml)
-        knownTreeSitterOrgGrammarRepos
         ignoredTreeSitterOrgRepos
         ;
+      knownTreeSitterOrgGrammarRepos =
+        map
+          ({repo, ...}: repo)
+          grammarsToml.knownTreeSitterOrgGrammarRepos;
     }
     (writers.writePython3 "updateImpl" {
         flakeIgnore = ["E501"];

From da6da251697a2dd858e8539c23569e29944b8f61 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Mon, 7 Nov 2022 11:01:00 +0100
Subject: [PATCH 11/14] tree-sitter/grammars: toml -> json and add schema
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The toml quickly got out of hand when I wanted to add more fields to
the list of known grammars, since then you need something like

```
[[knownTreeSitterOrgGrammarRepos.foo]]
repo = "foo"
[[knownTreeSitterOrgGrammarRepos.foo.grammars]]
name = "foo-grammar-bar"
location = "bar"
```

Just to express

```
{ knownTreeSitterOrgGrammarRepos = [
  { foo = {
      repo = "foo"
      grammars = [
        { name = "foo-grammar-bar"
          location = "bar"
        }
      ]
    }
  }
]}
```

Which translates to a *very* verbose config file.

Instead, let’s convert to json. Of course that means no comments are
possible, but there is a good solution that every modern editor
supports: json schema.
This also gives auto-completion and documentation for free.
---
 .../tools/parsing/tree-sitter/grammars.json   | 492 ++++++++++++++++++
 .../parsing/tree-sitter/grammars.schema.json  | 108 ++++
 .../tools/parsing/tree-sitter/grammars.toml   | 375 -------------
 .../tools/parsing/tree-sitter/update.nix      |  10 +-
 4 files changed, 605 insertions(+), 380 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars.schema.json
 delete mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars.toml

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.json b/pkgs/development/tools/parsing/tree-sitter/grammars.json
new file mode 100644
index 0000000000000..751b93770f072
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars.json
@@ -0,0 +1,492 @@
+{
+  "$schema": "./grammars.schema.json",
+  "ignoredTreeSitterOrgRepos": [
+    {
+      "repo": "tree-sitter",
+      "reason": "main implementation"
+    },
+    {
+      "repo": "tree-sitter-cli",
+      "reason": "cli implementation"
+    },
+    {
+      "repo": "haskell-tree-sitter",
+      "reason": "this is the haskell language bindings, tree-sitter-haskell is the grammar"
+    },
+    {
+      "repo": "ruby-tree-sitter.old",
+      "reason": "this is the ruby language bindings, tree-sitter-ruby is the grammar"
+    },
+    {
+      "repo": "rust-tree-sitter",
+      "reason": "this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar"
+    },
+    {
+      "repo": "node-tree-sitter",
+      "reason": "this is the nodejs language bindings, tree-sitter-javascript is the grammar"
+    },
+    {
+      "repo": "py-tree-sitter",
+      "reason": "this is the python language bindings, tree-sitter-python is the grammar"
+    },
+    {
+      "repo": "afl-tree-sitter",
+      "reason": "afl fuzzing for tree sitter"
+    },
+    {
+      "repo": "highlight-schema",
+      "reason": "archived"
+    },
+    {
+      "repo": "tree-sitter.github.io",
+      "reason": "website"
+    },
+    {
+      "repo": "tree-sitter-razor",
+      "reason": "not maintained"
+    },
+    {
+      "repo": "tree-sitter-graph",
+      "reason": "rust library for constructing arbitrary graph structures from source code"
+    },
+    {
+      "repo": "tree-sitter-swift",
+      "reason": "abandoned"
+    }
+  ],
+  "knownTreeSitterOrgGrammarRepos": [
+    {
+      "repo": "tree-sitter-javascript"
+    },
+    {
+      "repo": "tree-sitter-c"
+    },
+    {
+      "repo": "tree-sitter-json"
+    },
+    {
+      "repo": "tree-sitter-cpp"
+    },
+    {
+      "repo": "tree-sitter-ruby"
+    },
+    {
+      "repo": "tree-sitter-go"
+    },
+    {
+      "repo": "tree-sitter-c-sharp"
+    },
+    {
+      "repo": "tree-sitter-python"
+    },
+    {
+      "repo": "tree-sitter-typescript"
+    },
+    {
+      "repo": "tree-sitter-rust"
+    },
+    {
+      "repo": "tree-sitter-bash"
+    },
+    {
+      "repo": "tree-sitter-php"
+    },
+    {
+      "repo": "tree-sitter-java"
+    },
+    {
+      "repo": "tree-sitter-scala"
+    },
+    {
+      "repo": "tree-sitter-ocaml"
+    },
+    {
+      "repo": "tree-sitter-julia"
+    },
+    {
+      "repo": "tree-sitter-agda"
+    },
+    {
+      "repo": "tree-sitter-fluent"
+    },
+    {
+      "repo": "tree-sitter-html"
+    },
+    {
+      "repo": "tree-sitter-haskell"
+    },
+    {
+      "repo": "tree-sitter-regex"
+    },
+    {
+      "repo": "tree-sitter-css"
+    },
+    {
+      "repo": "tree-sitter-verilog"
+    },
+    {
+      "repo": "tree-sitter-jsdoc"
+    },
+    {
+      "repo": "tree-sitter-ql"
+    },
+    {
+      "repo": "tree-sitter-ql-dbscheme"
+    },
+    {
+      "repo": "tree-sitter-embedded-template"
+    },
+    {
+      "repo": "tree-sitter-tsq"
+    },
+    {
+      "repo": "tree-sitter-toml"
+    }
+  ],
+  "otherGrammars": {
+    "tree-sitter-beancount": {
+      "type": "github",
+      "orga": "polarmutex",
+      "repo": "tree-sitter-beancount"
+    },
+    "tree-sitter-clojure": {
+      "type": "github",
+      "orga": "sogaiu",
+      "repo": "tree-sitter-clojure"
+    },
+    "tree-sitter-comment": {
+      "type": "github",
+      "orga": "stsewd",
+      "repo": "tree-sitter-comment"
+    },
+    "tree-sitter-dart": {
+      "type": "github",
+      "orga": "usernobody14",
+      "repo": "tree-sitter-dart"
+    },
+    "tree-sitter-elisp": {
+      "type": "github",
+      "orga": "wilfred",
+      "repo": "tree-sitter-elisp"
+    },
+    "tree-sitter-nix": {
+      "type": "github",
+      "orga": "cstrahan",
+      "repo": "tree-sitter-nix"
+    },
+    "tree-sitter-latex": {
+      "type": "github",
+      "orga": "latex-lsp",
+      "repo": "tree-sitter-latex"
+    },
+    "tree-sitter-lua": {
+      "type": "github",
+      "orga": "MunifTanjim",
+      "repo": "tree-sitter-lua"
+    },
+    "tree-sitter-fennel": {
+      "type": "github",
+      "orga": "travonted",
+      "repo": "tree-sitter-fennel"
+    },
+    "tree-sitter-make": {
+      "type": "github",
+      "orga": "alemuller",
+      "repo": "tree-sitter-make"
+    },
+    "tree-sitter-markdown": {
+      "type": "github",
+      "orga": "MDeiml",
+      "repo": "tree-sitter-markdown"
+    },
+    "tree-sitter-rego": {
+      "type": "github",
+      "orga": "FallenAngel97",
+      "repo": "tree-sitter-rego"
+    },
+    "tree-sitter-rst": {
+      "type": "github",
+      "orga": "stsewd",
+      "repo": "tree-sitter-rst"
+    },
+    "tree-sitter-svelte": {
+      "type": "github",
+      "orga": "Himujjal",
+      "repo": "tree-sitter-svelte"
+    },
+    "tree-sitter-sql": {
+      "type": "github",
+      "orga": "m-novikov",
+      "repo": "tree-sitter-sql"
+    },
+    "tree-sitter-vim": {
+      "type": "github",
+      "orga": "vigoux",
+      "repo": "tree-sitter-viml"
+    },
+    "tree-sitter-yaml": {
+      "type": "github",
+      "orga": "ikatyang",
+      "repo": "tree-sitter-yaml"
+    },
+    "tree-sitter-zig": {
+      "type": "github",
+      "orga": "maxxnino",
+      "repo": "tree-sitter-zig"
+    },
+    "tree-sitter-fish": {
+      "type": "github",
+      "orga": "ram02z",
+      "repo": "tree-sitter-fish"
+    },
+    "tree-sitter-dot": {
+      "type": "github",
+      "orga": "rydesun",
+      "repo": "tree-sitter-dot"
+    },
+    "tree-sitter-norg": {
+      "type": "github",
+      "orga": "nvim-neorg",
+      "repo": "tree-sitter-norg"
+    },
+    "tree-sitter-commonlisp": {
+      "type": "github",
+      "orga": "thehamsta",
+      "repo": "tree-sitter-commonlisp"
+    },
+    "tree-sitter-cuda": {
+      "type": "github",
+      "orga": "thehamsta",
+      "repo": "tree-sitter-cuda"
+    },
+    "tree-sitter-glsl": {
+      "type": "github",
+      "orga": "thehamsta",
+      "repo": "tree-sitter-glsl"
+    },
+    "tree-sitter-dockerfile": {
+      "type": "github",
+      "orga": "camdencheek",
+      "repo": "tree-sitter-dockerfile"
+    },
+    "tree-sitter-ledger": {
+      "type": "github",
+      "orga": "cbarrete",
+      "repo": "tree-sitter-ledger"
+    },
+    "tree-sitter-gomod": {
+      "type": "github",
+      "orga": "camdencheek",
+      "repo": "tree-sitter-go-mod"
+    },
+    "tree-sitter-gowork": {
+      "type": "github",
+      "orga": "omertuc",
+      "repo": "tree-sitter-go-work"
+    },
+    "tree-sitter-graphql": {
+      "type": "github",
+      "orga": "bkegley",
+      "repo": "tree-sitter-graphql"
+    },
+    "tree-sitter-pgn": {
+      "type": "github",
+      "orga": "rolandwalker",
+      "repo": "tree-sitter-pgn"
+    },
+    "tree-sitter-perl": {
+      "type": "github",
+      "orga": "ganezdragon",
+      "repo": "tree-sitter-perl"
+    },
+    "tree-sitter-kotlin": {
+      "type": "github",
+      "orga": "fwcd",
+      "repo": "tree-sitter-kotlin"
+    },
+    "tree-sitter-scss": {
+      "type": "github",
+      "orga": "serenadeai",
+      "repo": "tree-sitter-scss"
+    },
+    "tree-sitter-erlang": {
+      "type": "github",
+      "orga": "abstractmachineslab",
+      "repo": "tree-sitter-erlang"
+    },
+    "tree-sitter-elixir": {
+      "type": "github",
+      "orga": "elixir-lang",
+      "repo": "tree-sitter-elixir"
+    },
+    "tree-sitter-surface": {
+      "type": "github",
+      "orga": "connorlay",
+      "repo": "tree-sitter-surface"
+    },
+    "tree-sitter-heex": {
+      "type": "github",
+      "orga": "connorlay",
+      "repo": "tree-sitter-heex"
+    },
+    "tree-sitter-supercollider": {
+      "type": "github",
+      "orga": "madskjeldgaard",
+      "repo": "tree-sitter-supercollider"
+    },
+    "tree-sitter-tlaplus": {
+      "type": "github",
+      "orga": "tlaplus-community",
+      "repo": "tree-sitter-tlaplus"
+    },
+    "tree-sitter-glimmer": {
+      "type": "github",
+      "orga": "alexlafroscia",
+      "repo": "tree-sitter-glimmer"
+    },
+    "tree-sitter-pug": {
+      "type": "github",
+      "orga": "zealot128",
+      "repo": "tree-sitter-pug"
+    },
+    "tree-sitter-vue": {
+      "type": "github",
+      "orga": "ikatyang",
+      "repo": "tree-sitter-vue"
+    },
+    "tree-sitter-elm": {
+      "type": "github",
+      "orga": "elm-tooling",
+      "repo": "tree-sitter-elm"
+    },
+    "tree-sitter-yang": {
+      "type": "github",
+      "orga": "hubro",
+      "repo": "tree-sitter-yang"
+    },
+    "tree-sitter-query": {
+      "type": "github",
+      "orga": "nvim-treesitter",
+      "repo": "tree-sitter-query"
+    },
+    "tree-sitter-sparql": {
+      "type": "github",
+      "orga": "bonabeavis",
+      "repo": "tree-sitter-sparql"
+    },
+    "tree-sitter-gdscript": {
+      "type": "github",
+      "orga": "prestonknopp",
+      "repo": "tree-sitter-gdscript"
+    },
+    "tree-sitter-godot-resource": {
+      "type": "github",
+      "orga": "prestonknopp",
+      "repo": "tree-sitter-godot-resource"
+    },
+    "tree-sitter-turtle": {
+      "type": "github",
+      "orga": "bonabeavis",
+      "repo": "tree-sitter-turtle"
+    },
+    "tree-sitter-devicetree": {
+      "type": "github",
+      "orga": "joelspadin",
+      "repo": "tree-sitter-devicetree"
+    },
+    "tree-sitter-r": {
+      "type": "github",
+      "orga": "r-lib",
+      "repo": "tree-sitter-r"
+    },
+    "tree-sitter-bibtex": {
+      "type": "github",
+      "orga": "latex-lsp",
+      "repo": "tree-sitter-bibtex"
+    },
+    "tree-sitter-fortran": {
+      "type": "github",
+      "orga": "stadelmanma",
+      "repo": "tree-sitter-fortran"
+    },
+    "tree-sitter-cmake": {
+      "type": "github",
+      "orga": "uyha",
+      "repo": "tree-sitter-cmake"
+    },
+    "tree-sitter-janet-simple": {
+      "type": "github",
+      "orga": "sogaiu",
+      "repo": "tree-sitter-janet-simple"
+    },
+    "tree-sitter-json5": {
+      "type": "github",
+      "orga": "joakker",
+      "repo": "tree-sitter-json5"
+    },
+    "tree-sitter-pioasm": {
+      "type": "github",
+      "orga": "leo60228",
+      "repo": "tree-sitter-pioasm"
+    },
+    "tree-sitter-hjson": {
+      "type": "github",
+      "orga": "winston0410",
+      "repo": "tree-sitter-hjson"
+    },
+    "tree-sitter-llvm": {
+      "type": "github",
+      "orga": "benwilliamgraham",
+      "repo": "tree-sitter-llvm"
+    },
+    "tree-sitter-http": {
+      "type": "github",
+      "orga": "ntbbloodbath",
+      "repo": "tree-sitter-http"
+    },
+    "tree-sitter-prisma": {
+      "type": "github",
+      "orga": "victorhqc",
+      "repo": "tree-sitter-prisma"
+    },
+    "tree-sitter-org-nvim": {
+      "type": "github",
+      "orga": "milisims",
+      "repo": "tree-sitter-org"
+    },
+    "tree-sitter-hcl": {
+      "type": "github",
+      "orga": "MichaHoffmann",
+      "repo": "tree-sitter-hcl"
+    },
+    "tree-sitter-scheme": {
+      "type": "github",
+      "orga": "6cdh",
+      "repo": "tree-sitter-scheme"
+    },
+    "tree-sitter-tiger": {
+      "type": "github",
+      "orga": "ambroisie",
+      "repo": "tree-sitter-tiger"
+    },
+    "tree-sitter-nickel": {
+      "type": "github",
+      "orga": "nickel-lang",
+      "repo": "tree-sitter-nickel"
+    },
+    "tree-sitter-smithy": {
+      "type": "github",
+      "orga": "indoorvivants",
+      "repo": "tree-sitter-smithy"
+    },
+    "tree-sitter-jsonnet": {
+      "type": "github",
+      "orga": "sourcegraph",
+      "repo": "tree-sitter-jsonnet"
+    },
+    "tree-sitter-jsonc": {
+      "type": "gitlab",
+      "projectId": "24426815"
+    }
+  }
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.schema.json b/pkgs/development/tools/parsing/tree-sitter/grammars.schema.json
new file mode 100644
index 0000000000000..5269c1b1c990d
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars.schema.json
@@ -0,0 +1,108 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "description": "Definition of all tree-sitter grammars we know of.\n\nUsed by both the update script and the default.nix to construct the list of grammars.\n\nGrammar list: https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md\n",
+  "type": "object",
+  "required": [
+    "ignoredTreeSitterOrgRepos"
+  ],
+  "properties": {
+    "ignoredTreeSitterOrgRepos": {
+      "description": "Repositories in the tree-sitter github org that we want to ignore.\nThese do not contain grammars.\n\nWe want to have a full overview of the tree-sitter organization, so that we can notice when a new grammar is added.\n",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "repo",
+          "reason"
+        ],
+        "properties": {
+          "repo": {
+            "description": "Repository in the tree-sitter github organization which should be ignored.",
+            "type": "string"
+          },
+          "reason": {
+            "description": "Reason why this repository is ignored.",
+            "type": "string"
+          }
+        }
+      }
+    },
+    "knownTreeSitterOrgGrammarRepos": {
+      "description": "Grammars we want to fetch from the tree-sitter github orga",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "repo"
+        ],
+        "properties": {
+          "repo": {
+            "description": "Repository name in the tree-sitter github organizaiton, containing a grammar. If there is no `grammars` field, this name will also be used as the name of the grammar & the nix attribute.",
+            "type": "string"
+          }
+        }
+      }
+    },
+    "otherGrammars": {
+      "description": "Additional grammars that are not in the official github orga.\nIf you need a grammar that already exists in the official orga, make sure to give it a different name.\n",
+      "type": "object",
+      "additionalProperties": {
+        "allOf": [
+          {
+            "description": "A grammar that does not live in the official tree-sitter github orga.",
+            "type": "object",
+            "required": [
+              "type"
+            ],
+            "properties": {
+              "type": {
+                "description": "The type of repo to fetch.",
+                "enum": [
+                  "github",
+                  "gitlab"
+                ]
+              }
+            }
+          },
+          {
+            "oneOf": [
+              {
+                "required": [
+                  "orga",
+                  "repo"
+                ],
+                "properties": {
+                  "type": {
+                    "const": "github"
+                  },
+                  "orga": {
+                    "description": "The github organization of this grammar’s repo",
+                    "type": "string"
+                  },
+                  "repo": {
+                    "description": "The github repo",
+                    "type": "string"
+                  }
+                }
+              },
+              {
+                "required": [
+                  "projectId"
+                ],
+                "properties": {
+                  "type": {
+                    "const": "gitlab"
+                  },
+                  "projectId": {
+                    "description": "The Gitlab projectId of this repository. It is displayed below the repository name on the web UI.",
+                    "type": "string"
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      }
+    }
+  }
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars.toml b/pkgs/development/tools/parsing/tree-sitter/grammars.toml
deleted file mode 100644
index 488cf5b810355..0000000000000
--- a/pkgs/development/tools/parsing/tree-sitter/grammars.toml
+++ /dev/null
@@ -1,375 +0,0 @@
-# Grammar list:
-# https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
-
-# repos of the tree-sitter github orga we want to ignore (not grammars)
-ignoredTreeSitterOrgRepos = [
-  "tree-sitter",
-  "tree-sitter-cli",
-  # this is the haskell language bindings, tree-sitter-haskell is the grammar
-  "haskell-tree-sitter",
-  # this is the ruby language bindings, tree-sitter-ruby is the grammar
-  "ruby-tree-sitter.old",
-  # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
-  "rust-tree-sitter",
-  # this is the nodejs language bindings, tree-sitter-javascript is the grammar
-  "node-tree-sitter",
-  # this is the python language bindings, tree-sitter-python is the grammar
-  "py-tree-sitter",
-  # afl fuzzing for tree sitter
-  "afl-tree-sitter",
-  # archived
-  "highlight-schema",
-  # website
-  "tree-sitter.github.io",
-  # not maintained
-  "tree-sitter-razor",
-  # rust library for constructing arbitrary graph structures from source code
-  "tree-sitter-graph",
-  # abandoned
-  "tree-sitter-swift",
-]
-
-# Grammars we want to fetch from the tree-sitter github orga
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-javascript"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-c"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-json"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-cpp"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-ruby"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-go"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-c-sharp"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-python"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-typescript"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-rust"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-bash"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-php"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-java"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-scala"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-ocaml"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-julia"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-agda"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-fluent"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-html"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-haskell"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-regex"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-css"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-verilog"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-jsdoc"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-ql"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-ql-dbscheme"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-embedded-template"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-tsq"
-[[knownTreeSitterOrgGrammarRepos]]
-repo = "tree-sitter-toml"
-
-
-# Additional grammars that are not in the official github orga.
-# If you need a grammar that already exists in the official orga,
-# make sure to give it a different name.
-#
-# This can be type == "github", then it needs an orga and a repo.
-#
-# Or it can be type == "gitlab", then it needs a projectId,
-# which you can find under the repository name on gitlab.
-[otherGrammars.tree-sitter-beancount]
-type = "github"
-orga = "polarmutex"
-repo = "tree-sitter-beancount"
-[otherGrammars.tree-sitter-clojure]
-type = "github"
-orga = "sogaiu"
-repo = "tree-sitter-clojure"
-[otherGrammars.tree-sitter-comment]
-type = "github"
-orga = "stsewd"
-repo = "tree-sitter-comment"
-[otherGrammars.tree-sitter-dart]
-type = "github"
-orga = "usernobody14"
-repo = "tree-sitter-dart"
-[otherGrammars.tree-sitter-elisp]
-type = "github"
-orga = "wilfred"
-repo = "tree-sitter-elisp"
-[otherGrammars.tree-sitter-nix]
-type = "github"
-orga = "cstrahan"
-repo = "tree-sitter-nix"
-[otherGrammars.tree-sitter-latex]
-type = "github"
-orga = "latex-lsp"
-repo = "tree-sitter-latex"
-[otherGrammars.tree-sitter-lua]
-type = "github"
-orga = "MunifTanjim"
-repo = "tree-sitter-lua"
-[otherGrammars.tree-sitter-fennel]
-type = "github"
-orga = "travonted"
-repo = "tree-sitter-fennel"
-[otherGrammars.tree-sitter-make]
-type = "github"
-orga = "alemuller"
-repo = "tree-sitter-make"
-[otherGrammars.tree-sitter-markdown]
-type = "github"
-orga = "MDeiml"
-repo = "tree-sitter-markdown"
-[otherGrammars.tree-sitter-rego]
-type = "github"
-orga = "FallenAngel97"
-repo = "tree-sitter-rego"
-[otherGrammars.tree-sitter-rst]
-type = "github"
-orga = "stsewd"
-repo = "tree-sitter-rst"
-[otherGrammars.tree-sitter-svelte]
-type = "github"
-orga = "Himujjal"
-repo = "tree-sitter-svelte"
-[otherGrammars.tree-sitter-sql]
-type = "github"
-orga = "m-novikov"
-repo = "tree-sitter-sql"
-[otherGrammars.tree-sitter-vim]
-type = "github"
-orga = "vigoux"
-repo = "tree-sitter-viml"
-[otherGrammars.tree-sitter-yaml]
-type = "github"
-orga = "ikatyang"
-repo = "tree-sitter-yaml"
-[otherGrammars.tree-sitter-zig]
-type = "github"
-orga = "maxxnino"
-repo = "tree-sitter-zig"
-[otherGrammars.tree-sitter-fish]
-type = "github"
-orga = "ram02z"
-repo = "tree-sitter-fish"
-[otherGrammars.tree-sitter-dot]
-type = "github"
-orga = "rydesun"
-repo = "tree-sitter-dot"
-[otherGrammars.tree-sitter-norg]
-type = "github"
-orga = "nvim-neorg"
-repo = "tree-sitter-norg"
-[otherGrammars.tree-sitter-commonlisp]
-type = "github"
-orga = "thehamsta"
-repo = "tree-sitter-commonlisp"
-[otherGrammars.tree-sitter-cuda]
-type = "github"
-orga = "thehamsta"
-repo = "tree-sitter-cuda"
-[otherGrammars.tree-sitter-glsl]
-type = "github"
-orga = "thehamsta"
-repo = "tree-sitter-glsl"
-[otherGrammars.tree-sitter-dockerfile]
-type = "github"
-orga = "camdencheek"
-repo = "tree-sitter-dockerfile"
-[otherGrammars.tree-sitter-ledger]
-type = "github"
-orga = "cbarrete"
-repo = "tree-sitter-ledger"
-[otherGrammars.tree-sitter-gomod]
-type = "github"
-orga = "camdencheek"
-repo = "tree-sitter-go-mod"
-[otherGrammars.tree-sitter-gowork]
-type = "github"
-orga = "omertuc"
-repo = "tree-sitter-go-work"
-[otherGrammars.tree-sitter-graphql]
-type = "github"
-orga = "bkegley"
-repo = "tree-sitter-graphql"
-[otherGrammars.tree-sitter-pgn]
-type = "github"
-orga = "rolandwalker"
-repo = "tree-sitter-pgn"
-[otherGrammars.tree-sitter-perl]
-type = "github"
-orga = "ganezdragon"
-repo = "tree-sitter-perl"
-[otherGrammars.tree-sitter-kotlin]
-type = "github"
-orga = "fwcd"
-repo = "tree-sitter-kotlin"
-[otherGrammars.tree-sitter-scss]
-type = "github"
-orga = "serenadeai"
-repo = "tree-sitter-scss"
-[otherGrammars.tree-sitter-erlang]
-type = "github"
-orga = "abstractmachineslab"
-repo = "tree-sitter-erlang"
-[otherGrammars.tree-sitter-elixir]
-type = "github"
-orga = "elixir-lang"
-repo = "tree-sitter-elixir"
-[otherGrammars.tree-sitter-surface]
-type = "github"
-orga = "connorlay"
-repo = "tree-sitter-surface"
-[otherGrammars.tree-sitter-heex]
-type = "github"
-orga = "connorlay"
-repo = "tree-sitter-heex"
-[otherGrammars.tree-sitter-supercollider]
-type = "github"
-orga = "madskjeldgaard"
-repo = "tree-sitter-supercollider"
-[otherGrammars.tree-sitter-tlaplus]
-type = "github"
-orga = "tlaplus-community"
-repo = "tree-sitter-tlaplus"
-[otherGrammars.tree-sitter-glimmer]
-type = "github"
-orga = "alexlafroscia"
-repo = "tree-sitter-glimmer"
-[otherGrammars.tree-sitter-pug]
-type = "github"
-orga = "zealot128"
-repo = "tree-sitter-pug"
-[otherGrammars.tree-sitter-vue]
-type = "github"
-orga = "ikatyang"
-repo = "tree-sitter-vue"
-[otherGrammars.tree-sitter-elm]
-type = "github"
-orga = "elm-tooling"
-repo = "tree-sitter-elm"
-[otherGrammars.tree-sitter-yang]
-type = "github"
-orga = "hubro"
-repo = "tree-sitter-yang"
-[otherGrammars.tree-sitter-query]
-type = "github"
-orga = "nvim-treesitter"
-repo = "tree-sitter-query"
-[otherGrammars.tree-sitter-sparql]
-type = "github"
-orga = "bonabeavis"
-repo = "tree-sitter-sparql"
-[otherGrammars.tree-sitter-gdscript]
-type = "github"
-orga = "prestonknopp"
-repo = "tree-sitter-gdscript"
-[otherGrammars.tree-sitter-godot-resource]
-type = "github"
-orga = "prestonknopp"
-repo = "tree-sitter-godot-resource"
-[otherGrammars.tree-sitter-turtle]
-type = "github"
-orga = "bonabeavis"
-repo = "tree-sitter-turtle"
-[otherGrammars.tree-sitter-devicetree]
-type = "github"
-orga = "joelspadin"
-repo = "tree-sitter-devicetree"
-[otherGrammars.tree-sitter-r]
-type = "github"
-orga = "r-lib"
-repo = "tree-sitter-r"
-[otherGrammars.tree-sitter-bibtex]
-type = "github"
-orga = "latex-lsp"
-repo = "tree-sitter-bibtex"
-[otherGrammars.tree-sitter-fortran]
-type = "github"
-orga = "stadelmanma"
-repo = "tree-sitter-fortran"
-[otherGrammars.tree-sitter-cmake]
-type = "github"
-orga = "uyha"
-repo = "tree-sitter-cmake"
-[otherGrammars.tree-sitter-janet-simple]
-type = "github"
-orga = "sogaiu"
-repo = "tree-sitter-janet-simple"
-[otherGrammars.tree-sitter-json5]
-type = "github"
-orga = "joakker"
-repo = "tree-sitter-json5"
-[otherGrammars.tree-sitter-pioasm]
-type = "github"
-orga = "leo60228"
-repo = "tree-sitter-pioasm"
-[otherGrammars.tree-sitter-hjson]
-type = "github"
-orga = "winston0410"
-repo = "tree-sitter-hjson"
-[otherGrammars.tree-sitter-llvm]
-type = "github"
-orga = "benwilliamgraham"
-repo = "tree-sitter-llvm"
-[otherGrammars.tree-sitter-http]
-type = "github"
-orga = "ntbbloodbath"
-repo = "tree-sitter-http"
-[otherGrammars.tree-sitter-prisma]
-type = "github"
-orga = "victorhqc"
-repo = "tree-sitter-prisma"
-[otherGrammars.tree-sitter-org-nvim]
-type = "github"
-orga = "milisims"
-repo = "tree-sitter-org"
-[otherGrammars.tree-sitter-hcl]
-type = "github"
-orga = "MichaHoffmann"
-repo = "tree-sitter-hcl"
-[otherGrammars.tree-sitter-scheme]
-type = "github"
-orga = "6cdh"
-repo = "tree-sitter-scheme"
-[otherGrammars.tree-sitter-tiger]
-type = "github"
-orga = "ambroisie"
-repo = "tree-sitter-tiger"
-[otherGrammars.tree-sitter-nickel]
-type = "github"
-orga = "nickel-lang"
-repo = "tree-sitter-nickel"
-[otherGrammars.tree-sitter-smithy]
-type = "github"
-orga = "indoorvivants"
-repo = "tree-sitter-smithy"
-[otherGrammars.tree-sitter-jsonnet]
-type = "github"
-orga = "sourcegraph"
-repo = "tree-sitter-jsonnet"
-[otherGrammars.tree-sitter-jsonc]
-type = "gitlab"
-projectId = "24426815"
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 0ccc9cf80bebd..f27b9aabe9d11 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -11,7 +11,7 @@
 
 
 let
-  grammarsToml = builtins.fromTOML (builtins.readFile ./grammars.toml);
+  grammarsJson = lib.importJSON ./grammars.json;
 
 
   # a list of {nixRepoAttrName, type, <type-specific>}
@@ -29,11 +29,11 @@ let
                 inherit repo;
               };
             })
-          grammarsToml.knownTreeSitterOrgGrammarRepos);
+          grammarsJson.knownTreeSitterOrgGrammarRepos);
 
       merged =
         mergeAttrsUnique
-          grammarsToml.otherGrammars
+          grammarsJson.otherGrammars
           treeSitterOrgaGrammars;
     in
       lib.mapAttrsToList
@@ -64,13 +64,13 @@ let
         nix-prefetch-git = "${nix-prefetch-git}/bin/nix-prefetch-git";
         printf = "${coreutils}/bin/printf";
       };
-      inherit (grammarsToml)
+      inherit (grammarsJson)
         ignoredTreeSitterOrgRepos
         ;
       knownTreeSitterOrgGrammarRepos =
         map
           ({repo, ...}: repo)
-          grammarsToml.knownTreeSitterOrgGrammarRepos;
+          grammarsJson.knownTreeSitterOrgGrammarRepos;
     }
     (writers.writePython3 "updateImpl" {
         flakeIgnore = ["E501"];

From 1bd586a26b4e49f20eac9ec05cc3e30e882968e3 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Mon, 7 Nov 2022 14:31:38 +0100
Subject: [PATCH 12/14] tree-sitter: reformulate `grammars` with lib.pipe

Looks like this is just an elaborate linkFarm command.
---
 .../tools/parsing/tree-sitter/default.nix          | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index af7ea347bdc6c..5334e87268953 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -41,12 +41,18 @@ let
 
   fetchGrammar = (v: fetchgit { inherit (v) url rev sha256 fetchSubmodules; });
 
+  # TODO: use linkFarm
   grammars =
-    runCommand "grammars" { } (''
+    runCommand "grammars" { } ''
       mkdir $out
-    '' + (lib.concatStrings (lib.mapAttrsToList
-      (name: grammar: "ln -s ${if grammar ? src then grammar.src else fetchGrammar grammar} $out/${name}\n")
-      (import ./grammars { inherit lib; }))));
+      ${lib.pipe
+         (import ./grammars { inherit lib; })
+         [
+          (lib.mapAttrsToList
+            (name: grammar: "ln -s ${if grammar ? src then grammar.src else fetchGrammar grammar} $out/${name}\n"))
+          lib.concatStrings
+         ]}
+    '';
 
   builtGrammars =
     let

From a4f7ec941096332d7d06038a2bc48a7661fdce41 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Mon, 7 Nov 2022 14:37:29 +0100
Subject: [PATCH 13/14] tree-sitter: reformulate `withPlugins` with lib.pipe

So much nicer :w:
---
 .../tools/parsing/tree-sitter/default.nix     | 26 +++++++++----------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 5334e87268953..09d8057e0db3a 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -82,25 +82,23 @@ let
   # which is equivalent to
   # pkgs.tree-sitter.withPlugins (p: builtins.attrValues p)
   withPlugins = grammarFn:
-    let
-      grammars = grammarFn builtGrammars;
-    in
-    linkFarm "grammars"
+    lib.pipe builtGrammars [
+      grammarFn
       (map
         (drv:
-          let
-            name = lib.strings.getName drv;
-          in
           {
-            name =
-              (lib.strings.replaceStrings [ "-" ] [ "_" ]
-                (lib.strings.removePrefix "tree-sitter-"
-                  (lib.strings.removeSuffix "-grammar" name)))
-              + ".so";
+            name = lib.pipe drv [
+              lib.strings.getName
+              (lib.strings.removeSuffix "-grammar")
+              (lib.strings.removePrefix "tree-sitter-")
+              (lib.strings.replaceStrings [ "-" ] [ "_" ])
+              (name: name + ".so")
+            ];
             path = "${drv}/parser";
           }
-        )
-        grammars);
+        ))
+      (linkFarm "grammars")
+    ];
 
   allGrammars = builtins.attrValues builtGrammars;
 

From 9532452c6233de875e051c8880d6e07c4c141e2e Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Mon, 7 Nov 2022 14:54:07 +0100
Subject: [PATCH 14/14] tree-sitter: remove support for the extraGrammars
 attribute
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This argument was extremely underspecified and hacked into the rest of
the code. I’d rather remove it for now and re-do support later if
necessary.

Ideally we’d expose the building blocks in a defined & documented
fashion.
---
 .../tools/parsing/tree-sitter/default.nix            | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 09d8057e0db3a..306d697ead2e9 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -20,9 +20,13 @@
 , enableShared ? !stdenv.hostPlatform.isStatic
 , enableStatic ? stdenv.hostPlatform.isStatic
 , webUISupport ? false
+
+# REMOVED
 , extraGrammars ? { }
 }:
 
+assert lib.assertMsg (extraGrammars == {}) "The `extraGrammars` for tree-sitter was removed, because the schema of extraGrammars was underspecified & undocumented. If you need support, please open an issue and ping @Profpatsch";
+
 let
   # to update: see ./README.md
   version = "0.20.7";
@@ -42,6 +46,8 @@ let
   fetchGrammar = (v: fetchgit { inherit (v) url rev sha256 fetchSubmodules; });
 
   # TODO: use linkFarm
+
+  # All grammar definitions’ source repositories.
   grammars =
     runCommand "grammars" { } ''
       mkdir $out
@@ -49,7 +55,7 @@ let
          (import ./grammars { inherit lib; })
          [
           (lib.mapAttrsToList
-            (name: grammar: "ln -s ${if grammar ? src then grammar.src else fetchGrammar grammar} $out/${name}\n"))
+            (name: grammar: "ln -s ${fetchGrammar grammar} $out/${name}\n"))
           lib.concatStrings
          ]}
     '';
@@ -60,10 +66,10 @@ let
         callPackage ./build-grammar.nix { } {
           language = name;
           inherit version;
-          source = if grammar ? src then grammar.src else fetchGrammar grammar;
+          source = fetchGrammar grammar;
           location = if grammar ? location then grammar.location else null;
         };
-      grammars' = import ./grammars { inherit lib; } // extraGrammars;
+      grammars' = import ./grammars { inherit lib; };
       grammars = grammars' //
         { tree-sitter-ocaml = grammars'.tree-sitter-ocaml // { location = "ocaml"; }; } //
         { tree-sitter-ocaml-interface = grammars'.tree-sitter-ocaml // { location = "interface"; }; } //